mcp-efficiency-engine 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release-npm.yml +16 -7
- package/.vscode/mcp.json +12 -5
- package/README.md +16 -0
- package/bin/mcpee.js +118 -0
- package/package.json +2 -2
- package/scripts/ops/emit-terminal-command-telemetry.py +84 -0
- package/scripts/ops/install-terminal-telemetry-hook.ps1 +152 -0
- package/scripts/ops/migrate-localdb-to-azure-sql.ps1 +98 -0
- package/scripts/ops/publish-langsmith-kpis.py +295 -0
- package/scripts/ops/start-codegraph-mcp.ps1 +67 -0
- package/scripts/ops/start-gitnexus-mcp.ps1 +85 -0
- package/scripts/ops/trace-command.py +69 -0
- package/scripts/ops/uninstall-terminal-telemetry-hook.ps1 +53 -0
|
@@ -26,7 +26,7 @@ jobs:
|
|
|
26
26
|
- name: Setup Node.js
|
|
27
27
|
uses: actions/setup-node@v4
|
|
28
28
|
with:
|
|
29
|
-
node-version:
|
|
29
|
+
node-version: 22
|
|
30
30
|
registry-url: https://registry.npmjs.org
|
|
31
31
|
|
|
32
32
|
- name: Resolve publish version
|
|
@@ -94,14 +94,23 @@ jobs:
|
|
|
94
94
|
env:
|
|
95
95
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
96
96
|
|
|
97
|
-
- name:
|
|
97
|
+
- name: Record package.json sync requirement
|
|
98
98
|
if: steps.version.outputs.changed == 'true'
|
|
99
99
|
run: |
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
100
|
+
echo "package.json was bumped in CI to publish an available npm version (${{ steps.version.outputs.tag }})."
|
|
101
|
+
echo "Repository sync is skipped because main is protected and requires PR-based updates."
|
|
102
|
+
echo "Open a follow-up PR to align package.json with npm published version."
|
|
103
|
+
|
|
104
|
+
- name: Release summary
|
|
105
|
+
run: |
|
|
106
|
+
echo "## NPM Release" >> "$GITHUB_STEP_SUMMARY"
|
|
107
|
+
echo "- mode: ${{ steps.version.outputs.mode }}" >> "$GITHUB_STEP_SUMMARY"
|
|
108
|
+
echo "- published: ${{ steps.version.outputs.tag }}" >> "$GITHUB_STEP_SUMMARY"
|
|
109
|
+
if [ "${{ steps.version.outputs.changed }}" = "true" ]; then
|
|
110
|
+
echo "- package.json sync: skipped (protected branch, requires PR)" >> "$GITHUB_STEP_SUMMARY"
|
|
111
|
+
else
|
|
112
|
+
echo "- package.json sync: not required" >> "$GITHUB_STEP_SUMMARY"
|
|
113
|
+
fi
|
|
105
114
|
|
|
106
115
|
- name: Create GitHub release
|
|
107
116
|
uses: softprops/action-gh-release@v2
|
package/.vscode/mcp.json
CHANGED
|
@@ -7,16 +7,23 @@
|
|
|
7
7
|
"command": "codebase-memory-mcp"
|
|
8
8
|
},
|
|
9
9
|
"codegraph": {
|
|
10
|
-
"command": "
|
|
10
|
+
"command": "pwsh",
|
|
11
11
|
"args": [
|
|
12
|
-
"
|
|
13
|
-
"
|
|
12
|
+
"-NoProfile",
|
|
13
|
+
"-ExecutionPolicy",
|
|
14
|
+
"Bypass",
|
|
15
|
+
"-File",
|
|
16
|
+
"scripts/ops/start-codegraph-mcp.ps1"
|
|
14
17
|
]
|
|
15
18
|
},
|
|
16
19
|
"gitnexus": {
|
|
17
|
-
"command": "
|
|
20
|
+
"command": "pwsh",
|
|
18
21
|
"args": [
|
|
19
|
-
"
|
|
22
|
+
"-NoProfile",
|
|
23
|
+
"-ExecutionPolicy",
|
|
24
|
+
"Bypass",
|
|
25
|
+
"-File",
|
|
26
|
+
"scripts/ops/start-gitnexus-mcp.ps1"
|
|
20
27
|
],
|
|
21
28
|
"env": {
|
|
22
29
|
"GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT": "50000",
|
package/README.md
CHANGED
|
@@ -233,6 +233,17 @@ py -3 .\scripts\intake\run-routing-evals.py
|
|
|
233
233
|
.\scripts\ops\bye.ps1
|
|
234
234
|
```
|
|
235
235
|
|
|
236
|
+
Telemetría de terminal (PowerShell, opcional):
|
|
237
|
+
|
|
238
|
+
```powershell
|
|
239
|
+
mcpee observe-on
|
|
240
|
+
# ... comandos interactivos ...
|
|
241
|
+
mcpee observe-off
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
- `mcpee observe-on` instala un hook global de perfil PowerShell para emitir un evento de telemetría por comando usando `scripts/ops/emit-terminal-command-telemetry.py`.
|
|
245
|
+
- `mcpee observe-off` elimina el hook global y restaura el perfil sin instrumentación.
|
|
246
|
+
|
|
236
247
|
Validación extendida recomendada:
|
|
237
248
|
|
|
238
249
|
```powershell
|
|
@@ -253,6 +264,11 @@ Comportamiento del hook:
|
|
|
253
264
|
- `scripts/learning/iteration-value-report.py`
|
|
254
265
|
- `scripts/ops/publish-langsmith-kpis.py` (best effort)
|
|
255
266
|
|
|
267
|
+
Notas operativas recientes:
|
|
268
|
+
|
|
269
|
+
- `mcpee` envuelve scripts PowerShell con `scripts/ops/trace-command.py` para emitir trazas por comando (`mcpee.*`) en el collector.
|
|
270
|
+
- `scripts/ops/publish-langsmith-kpis.py` agrega snapshots locales de flujos, coste y tokens antes de publicar KPI runs en LangSmith.
|
|
271
|
+
|
|
256
272
|
Artefactos/resultados:
|
|
257
273
|
|
|
258
274
|
- AutoDocs actualizado en `autodocs/generated` y `autodocs/site`
|
package/bin/mcpee.js
CHANGED
|
@@ -12,8 +12,77 @@ const commandMap = {
|
|
|
12
12
|
hi: "scripts/ops/hi.ps1",
|
|
13
13
|
bye: "scripts/ops/bye.ps1",
|
|
14
14
|
intake: "scripts/intake/run-repo-intake.ps1",
|
|
15
|
+
"observe-on": "scripts/ops/install-terminal-telemetry-hook.ps1",
|
|
16
|
+
"observe-off": "scripts/ops/uninstall-terminal-telemetry-hook.ps1",
|
|
15
17
|
};
|
|
16
18
|
|
|
19
|
+
function resolveSessionId() {
|
|
20
|
+
const explicitSessionId = (process.env.MCP_SESSION_ID || "").trim();
|
|
21
|
+
if (explicitSessionId) {
|
|
22
|
+
return explicitSessionId;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const sessionLogHint = (process.env.VSCODE_TARGET_SESSION_LOG || "").trim();
|
|
26
|
+
if (sessionLogHint) {
|
|
27
|
+
const match = sessionLogHint.match(/[0-9a-fA-F-]{36}/);
|
|
28
|
+
if (match && match[0]) {
|
|
29
|
+
return match[0];
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return "mcpee-cli";
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function resolvePythonCandidates() {
|
|
37
|
+
if (process.platform === "win32") {
|
|
38
|
+
return [
|
|
39
|
+
["py", ["-3"]],
|
|
40
|
+
["python", []],
|
|
41
|
+
["python3", []],
|
|
42
|
+
];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return [
|
|
46
|
+
["python3", []],
|
|
47
|
+
["python", []],
|
|
48
|
+
];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function runCopilotUsageIngest(repoRootPath) {
|
|
52
|
+
const sessionLogHint = (process.env.VSCODE_TARGET_SESSION_LOG || "").trim();
|
|
53
|
+
if (!sessionLogHint) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const ingestScriptPath = path.join(repoRootPath, "scripts", "learning", "ingest-copilot-session-usage.py");
|
|
58
|
+
if (!fs.existsSync(ingestScriptPath)) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
for (const [pythonCommand, pythonPrefixArgs] of resolvePythonCandidates()) {
|
|
63
|
+
const ingestResult = spawnSync(
|
|
64
|
+
pythonCommand,
|
|
65
|
+
[
|
|
66
|
+
...pythonPrefixArgs,
|
|
67
|
+
ingestScriptPath,
|
|
68
|
+
"--session-log",
|
|
69
|
+
sessionLogHint,
|
|
70
|
+
],
|
|
71
|
+
{
|
|
72
|
+
cwd: repoRootPath,
|
|
73
|
+
stdio: "ignore",
|
|
74
|
+
},
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
if (ingestResult.error && ingestResult.error.code === "ENOENT") {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// La ingesta es best-effort: si falla no bloquea el comando principal.
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
17
86
|
function resolveExecutionRoot(scriptRelativePath) {
|
|
18
87
|
const configuredTarget = process.env.MCPEE_TARGET_DIR ? path.resolve(process.env.MCPEE_TARGET_DIR) : "";
|
|
19
88
|
const cwdRoot = process.cwd();
|
|
@@ -41,6 +110,8 @@ function printHelp() {
|
|
|
41
110
|
" hi Ejecuta el preflight diario.",
|
|
42
111
|
" bye Ejecuta el cierre diario.",
|
|
43
112
|
" intake Ejecuta el repo intake.",
|
|
113
|
+
" observe-on Activa telemetria global del terminal (hook PowerShell).",
|
|
114
|
+
" observe-off Desactiva telemetria global del terminal.",
|
|
44
115
|
"",
|
|
45
116
|
"Ejemplos:",
|
|
46
117
|
" npx mcp-efficiency-engine bootstrap",
|
|
@@ -60,6 +131,11 @@ function resolveInvocationCandidates() {
|
|
|
60
131
|
function runPowerShellScript(scriptRelativePath, forwardedArgs) {
|
|
61
132
|
const executionRoot = resolveExecutionRoot(scriptRelativePath);
|
|
62
133
|
const scriptPath = path.join(executionRoot, scriptRelativePath);
|
|
134
|
+
const tracerScriptPath = path.join(repoRoot, "scripts", "ops", "trace-command.py");
|
|
135
|
+
const sessionId = resolveSessionId();
|
|
136
|
+
|
|
137
|
+
runCopilotUsageIngest(repoRoot);
|
|
138
|
+
|
|
63
139
|
if (!fs.existsSync(scriptPath)) {
|
|
64
140
|
process.stderr.write(`Script no encontrado: ${scriptRelativePath}\n`);
|
|
65
141
|
return 1;
|
|
@@ -67,6 +143,48 @@ function runPowerShellScript(scriptRelativePath, forwardedArgs) {
|
|
|
67
143
|
|
|
68
144
|
const shellCandidates = resolveInvocationCandidates();
|
|
69
145
|
for (const shellCommand of shellCandidates) {
|
|
146
|
+
if (fs.existsSync(tracerScriptPath)) {
|
|
147
|
+
const operation = `mcpee.${path.basename(scriptRelativePath, path.extname(scriptRelativePath))}`;
|
|
148
|
+
for (const [pythonCommand, pythonPrefixArgs] of resolvePythonCandidates()) {
|
|
149
|
+
const tracedResult = spawnSync(
|
|
150
|
+
pythonCommand,
|
|
151
|
+
[
|
|
152
|
+
...pythonPrefixArgs,
|
|
153
|
+
tracerScriptPath,
|
|
154
|
+
"--operation",
|
|
155
|
+
operation,
|
|
156
|
+
"--session-id",
|
|
157
|
+
sessionId,
|
|
158
|
+
"--cwd",
|
|
159
|
+
executionRoot,
|
|
160
|
+
"--",
|
|
161
|
+
shellCommand,
|
|
162
|
+
"-NoProfile",
|
|
163
|
+
"-ExecutionPolicy",
|
|
164
|
+
"Bypass",
|
|
165
|
+
"-File",
|
|
166
|
+
scriptPath,
|
|
167
|
+
...forwardedArgs,
|
|
168
|
+
],
|
|
169
|
+
{
|
|
170
|
+
cwd: repoRoot,
|
|
171
|
+
stdio: "inherit",
|
|
172
|
+
},
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
if (tracedResult.error && tracedResult.error.code === "ENOENT") {
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (tracedResult.error) {
|
|
180
|
+
process.stderr.write(`${tracedResult.error.message}\n`);
|
|
181
|
+
return 1;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return tracedResult.status ?? 0;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
70
188
|
const result = spawnSync(
|
|
71
189
|
shellCommand,
|
|
72
190
|
["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", scriptPath, ...forwardedArgs],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mcp-efficiency-engine",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "Motor de orquestacion para agentes MCP con routing por dominio y bootstrap portable.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
"hi": "node ./bin/mcpee.js hi",
|
|
52
52
|
"bye": "node ./bin/mcpee.js bye",
|
|
53
53
|
"intake": "node ./bin/mcpee.js intake",
|
|
54
|
-
"langsmith:kpis": "py -3 ./scripts/ops/publish-langsmith-kpis.py",
|
|
54
|
+
"langsmith:kpis": "py -3 ./scripts/ops/trace-command.py --operation mcpee.langsmith-kpis --session-id mcpee-cli --cwd . -- py -3 ./scripts/ops/publish-langsmith-kpis.py",
|
|
55
55
|
"pack:check": "npm pack --dry-run"
|
|
56
56
|
},
|
|
57
57
|
"engines": {
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _bootstrap_repo_root() -> Path:
|
|
10
|
+
repo_root = Path(__file__).resolve().parents[2]
|
|
11
|
+
if str(repo_root) not in sys.path:
|
|
12
|
+
sys.path.insert(0, str(repo_root))
|
|
13
|
+
return repo_root
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _hash_command(command: str) -> str:
|
|
17
|
+
return hashlib.sha256(command.encode("utf-8")).hexdigest()[:16]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main() -> int:
|
|
21
|
+
parser = argparse.ArgumentParser(description="Emit telemetry for an already executed terminal command.")
|
|
22
|
+
parser.add_argument("--session-id", default="terminal-session", help="Session identifier")
|
|
23
|
+
parser.add_argument("--cwd", default=".", help="Current working directory")
|
|
24
|
+
parser.add_argument("--command", default="", help="Executed command text")
|
|
25
|
+
parser.add_argument("--success", default="true", help="Whether command succeeded (true/false)")
|
|
26
|
+
parser.add_argument("--exit-code", type=int, default=0, help="Exit code for native commands")
|
|
27
|
+
args = parser.parse_args()
|
|
28
|
+
|
|
29
|
+
command_text = (args.command or "").strip()
|
|
30
|
+
if not command_text:
|
|
31
|
+
return 0
|
|
32
|
+
|
|
33
|
+
repo_root = _bootstrap_repo_root()
|
|
34
|
+
from telemetry import build_telemetry_collector
|
|
35
|
+
|
|
36
|
+
collector = build_telemetry_collector(repo_root)
|
|
37
|
+
|
|
38
|
+
success = str(args.success).strip().lower() in {"1", "true", "yes", "ok"}
|
|
39
|
+
normalized_cwd = str(Path(args.cwd).resolve())
|
|
40
|
+
|
|
41
|
+
with collector.start_execution(
|
|
42
|
+
operation="terminal.command",
|
|
43
|
+
session_id=str(args.session_id).strip() or "terminal-session",
|
|
44
|
+
model="powershell",
|
|
45
|
+
provider="local-shell",
|
|
46
|
+
):
|
|
47
|
+
with collector.start_span(
|
|
48
|
+
name="terminal.executed_command",
|
|
49
|
+
kind="TOOL",
|
|
50
|
+
attributes={
|
|
51
|
+
"cwd": normalized_cwd,
|
|
52
|
+
"command_hash": _hash_command(command_text),
|
|
53
|
+
"success": success,
|
|
54
|
+
"exit_code": int(args.exit_code),
|
|
55
|
+
},
|
|
56
|
+
):
|
|
57
|
+
collector.record_event(
|
|
58
|
+
"RoutingResolved",
|
|
59
|
+
{
|
|
60
|
+
"source": "terminal-hook",
|
|
61
|
+
"cwd": normalized_cwd,
|
|
62
|
+
"success": success,
|
|
63
|
+
"exit_code": int(args.exit_code),
|
|
64
|
+
},
|
|
65
|
+
level="INFO",
|
|
66
|
+
)
|
|
67
|
+
collector.record_metric("terminal_command_count", 1.0, unit="count")
|
|
68
|
+
collector.record_metric("terminal_last_exit_code", float(int(args.exit_code)), unit="code")
|
|
69
|
+
if not success:
|
|
70
|
+
collector.record_event(
|
|
71
|
+
"ExceptionThrown",
|
|
72
|
+
{
|
|
73
|
+
"source": "terminal-hook",
|
|
74
|
+
"message": "command_reported_failure",
|
|
75
|
+
"exit_code": int(args.exit_code),
|
|
76
|
+
},
|
|
77
|
+
level="ERROR",
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return 0
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
if __name__ == "__main__":
|
|
84
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
param(
|
|
2
|
+
[switch]$AllHosts,
|
|
3
|
+
[switch]$CurrentProcess,
|
|
4
|
+
[string]$SessionId = ''
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
Set-StrictMode -Version Latest
|
|
8
|
+
$ErrorActionPreference = 'Stop'
|
|
9
|
+
|
|
10
|
+
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
|
|
11
|
+
$sessionIdResolved = if ([string]::IsNullOrWhiteSpace($SessionId)) {
|
|
12
|
+
if (-not [string]::IsNullOrWhiteSpace($env:MCP_SESSION_ID)) { $env:MCP_SESSION_ID }
|
|
13
|
+
elseif (-not [string]::IsNullOrWhiteSpace($env:VSCODE_TARGET_SESSION_LOG) -and $env:VSCODE_TARGET_SESSION_LOG -match '([0-9a-fA-F-]{36})') { $Matches[1] }
|
|
14
|
+
else { 'terminal-session' }
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
$SessionId
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
$hookBlock = @"
|
|
21
|
+
# >>> MCPEE_TERMINAL_TELEMETRY_START >>>
|
|
22
|
+
`$global:MCPEE_TELEMETRY_REPO_ROOT = '$repoRoot'
|
|
23
|
+
`$global:MCPEE_TELEMETRY_SESSION_ID = '$sessionIdResolved'
|
|
24
|
+
`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID = -1
|
|
25
|
+
`$global:MCPEE_TELEMETRY_LAST_STAMP = ''
|
|
26
|
+
|
|
27
|
+
function global:Get-McpeeTelemetryPythonCommand {
|
|
28
|
+
if (Get-Command py -ErrorAction SilentlyContinue) {
|
|
29
|
+
return @('py', '-3')
|
|
30
|
+
}
|
|
31
|
+
if (Get-Command python -ErrorAction SilentlyContinue) {
|
|
32
|
+
return @('python')
|
|
33
|
+
}
|
|
34
|
+
if (Get-Command python3 -ErrorAction SilentlyContinue) {
|
|
35
|
+
return @('python3')
|
|
36
|
+
}
|
|
37
|
+
return @()
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function global:Invoke-McpeeTerminalTelemetry {
|
|
41
|
+
try {
|
|
42
|
+
`$last = Get-History -Count 1 -ErrorAction SilentlyContinue
|
|
43
|
+
if (`$null -eq `$last) { return }
|
|
44
|
+
|
|
45
|
+
`$historyId = [int]`$last.Id
|
|
46
|
+
if (`$historyId -le [int]`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID) { return }
|
|
47
|
+
|
|
48
|
+
`$commandText = [string]`$last.CommandLine
|
|
49
|
+
if ([string]::IsNullOrWhiteSpace(`$commandText)) { return }
|
|
50
|
+
|
|
51
|
+
`$commandTrim = `$commandText.Trim()
|
|
52
|
+
if (`$commandTrim -match 'Invoke-McpeeTerminalTelemetry|emit-terminal-command-telemetry\.py') {
|
|
53
|
+
`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID = `$historyId
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
`$stamp = ('{0}:{1}' -f `$historyId, `$commandTrim)
|
|
58
|
+
if (`$stamp -eq `$global:MCPEE_TELEMETRY_LAST_STAMP) {
|
|
59
|
+
`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID = `$historyId
|
|
60
|
+
return
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
`$py = Get-McpeeTelemetryPythonCommand
|
|
64
|
+
if (`$py.Count -eq 0) {
|
|
65
|
+
`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID = `$historyId
|
|
66
|
+
`$global:MCPEE_TELEMETRY_LAST_STAMP = `$stamp
|
|
67
|
+
return
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
`$scriptPath = Join-Path `$global:MCPEE_TELEMETRY_REPO_ROOT 'scripts/ops/emit-terminal-command-telemetry.py'
|
|
71
|
+
if (-not (Test-Path `$scriptPath)) {
|
|
72
|
+
`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID = `$historyId
|
|
73
|
+
`$global:MCPEE_TELEMETRY_LAST_STAMP = `$stamp
|
|
74
|
+
return
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
`$success = if (`$?) { 'true' } else { 'false' }
|
|
78
|
+
`$nativeExit = if (`$null -eq `$LASTEXITCODE) { 0 } else { [int]`$LASTEXITCODE }
|
|
79
|
+
|
|
80
|
+
`$pyArgs = @()
|
|
81
|
+
if (`$py.Count -gt 1) {
|
|
82
|
+
`$pyArgs = `$py[1..(`$py.Count-1)]
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
& `$py[0] @(`$pyArgs) `$scriptPath --session-id `$global:MCPEE_TELEMETRY_SESSION_ID --cwd `$PWD.Path --command `$commandTrim --success `$success --exit-code `$nativeExit *> `$null
|
|
86
|
+
|
|
87
|
+
`$global:MCPEE_TELEMETRY_LAST_HISTORY_ID = `$historyId
|
|
88
|
+
`$global:MCPEE_TELEMETRY_LAST_STAMP = `$stamp
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
# Hook de telemetria best-effort: nunca romper la consola.
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (-not (Test-Path Function:\global:MCPEE_ORIGINAL_PROMPT)) {
|
|
96
|
+
`$originalPromptBody = if (Test-Path Function:\global:prompt) { (Get-Command prompt -CommandType Function).ScriptBlock.ToString() } else { '"PS " + `$executionContext.SessionState.Path.CurrentLocation + "> "' }
|
|
97
|
+
Set-Item -Path Function:\global:MCPEE_ORIGINAL_PROMPT -Value ([ScriptBlock]::Create(`$originalPromptBody)) -Force
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function global:prompt {
|
|
101
|
+
Invoke-McpeeTerminalTelemetry
|
|
102
|
+
& global:MCPEE_ORIGINAL_PROMPT
|
|
103
|
+
}
|
|
104
|
+
# <<< MCPEE_TERMINAL_TELEMETRY_END <<<
|
|
105
|
+
"@
|
|
106
|
+
|
|
107
|
+
function Install-InProfile {
|
|
108
|
+
param([string]$ProfilePath)
|
|
109
|
+
|
|
110
|
+
$dir = Split-Path $ProfilePath -Parent
|
|
111
|
+
if (-not (Test-Path $dir)) {
|
|
112
|
+
New-Item -ItemType Directory -Path $dir -Force | Out-Null
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (-not (Test-Path $ProfilePath)) {
|
|
116
|
+
New-Item -ItemType File -Path $ProfilePath -Force | Out-Null
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
$content = Get-Content -Path $ProfilePath -Raw -ErrorAction SilentlyContinue
|
|
120
|
+
if ($null -eq $content) { $content = '' }
|
|
121
|
+
|
|
122
|
+
if ($content -match 'MCPEE_TERMINAL_TELEMETRY_START') {
|
|
123
|
+
$updated = [regex]::Replace(
|
|
124
|
+
$content,
|
|
125
|
+
'(?s)# >>> MCPEE_TERMINAL_TELEMETRY_START >>>.*?# <<< MCPEE_TERMINAL_TELEMETRY_END <<<',
|
|
126
|
+
$hookBlock
|
|
127
|
+
)
|
|
128
|
+
Set-Content -Path $ProfilePath -Value $updated -Encoding UTF8
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
if (-not [string]::IsNullOrWhiteSpace($content) -and -not $content.EndsWith("`n")) {
|
|
132
|
+
$content += "`r`n"
|
|
133
|
+
}
|
|
134
|
+
$content += "`r`n$hookBlock`r`n"
|
|
135
|
+
Set-Content -Path $ProfilePath -Value $content -Encoding UTF8
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
Write-Host "[ok] Hook instalado/actualizado en: $ProfilePath"
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if ($CurrentProcess) {
|
|
142
|
+
Invoke-Expression $hookBlock
|
|
143
|
+
Write-Host '[ok] Hook cargado en la sesion actual.'
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
Install-InProfile -ProfilePath $PROFILE.CurrentUserCurrentHost
|
|
147
|
+
if ($AllHosts) {
|
|
148
|
+
Install-InProfile -ProfilePath $PROFILE.CurrentUserAllHosts
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
Write-Host '[ok] Telemetria global de terminal activada (best-effort).'
|
|
152
|
+
Write-Host '[hint] Para desinstalar usa: mcpee observe-off'
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
param(
|
|
2
|
+
[string]$SourceConnectionString = 'Server=(localdb)\MSSQLLocalDB;Database=TechRidersDev;Trusted_Connection=True;MultipleActiveResultSets=True;TrustServerCertificate=True',
|
|
3
|
+
[string]$TargetServer = 'sql-techriders-dev-ohbnpr4l4fihc.database.windows.net',
|
|
4
|
+
[string]$TargetDatabase = 'sqldb-techriders-dev',
|
|
5
|
+
[string]$TargetUser = 'sqladminuser'
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
$ErrorActionPreference = 'Stop'
|
|
9
|
+
Add-Type -AssemblyName System.Data
|
|
10
|
+
|
|
11
|
+
if ([string]::IsNullOrWhiteSpace($env:SQL_ADMIN_PASSWORD)) {
|
|
12
|
+
throw 'Missing SQL_ADMIN_PASSWORD environment variable.'
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
$targetConnectionString = "Server=tcp:$TargetServer,1433;Initial Catalog=$TargetDatabase;Persist Security Info=False;User ID=$TargetUser;Password=$($env:SQL_ADMIN_PASSWORD);MultipleActiveResultSets=True;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
|
|
16
|
+
|
|
17
|
+
$sourceConn = New-Object System.Data.SqlClient.SqlConnection $SourceConnectionString
|
|
18
|
+
$targetConn = New-Object System.Data.SqlClient.SqlConnection $targetConnectionString
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
$sourceConn.Open()
|
|
22
|
+
$targetConn.Open()
|
|
23
|
+
|
|
24
|
+
$tableCmd = $sourceConn.CreateCommand()
|
|
25
|
+
$tableCmd.CommandText = @"
|
|
26
|
+
SELECT s.name AS SchemaName, t.name AS TableName
|
|
27
|
+
FROM sys.tables t
|
|
28
|
+
JOIN sys.schemas s ON s.schema_id = t.schema_id
|
|
29
|
+
WHERE t.is_ms_shipped = 0
|
|
30
|
+
ORDER BY s.name, t.name;
|
|
31
|
+
"@
|
|
32
|
+
|
|
33
|
+
$reader = $tableCmd.ExecuteReader()
|
|
34
|
+
$tables = New-Object System.Collections.Generic.List[object]
|
|
35
|
+
while ($reader.Read()) {
|
|
36
|
+
$schema = $reader.GetString(0)
|
|
37
|
+
$table = $reader.GetString(1)
|
|
38
|
+
if ($table -ne '__EFMigrationsHistory') {
|
|
39
|
+
$tables.Add([PSCustomObject]@{ Schema = $schema; Table = $table })
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
$reader.Close()
|
|
43
|
+
|
|
44
|
+
if ($tables.Count -eq 0) {
|
|
45
|
+
throw 'No source user tables found.'
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
foreach ($t in $tables) {
|
|
49
|
+
$fullName = "[$($t.Schema)].[$($t.Table)]"
|
|
50
|
+
$disableCmd = $targetConn.CreateCommand()
|
|
51
|
+
$disableCmd.CommandText = "ALTER TABLE $fullName NOCHECK CONSTRAINT ALL;"
|
|
52
|
+
[void]$disableCmd.ExecuteNonQuery()
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
$summary = New-Object System.Collections.Generic.List[object]
|
|
56
|
+
|
|
57
|
+
foreach ($t in $tables) {
|
|
58
|
+
$fullName = "[$($t.Schema)].[$($t.Table)]"
|
|
59
|
+
|
|
60
|
+
$deleteCmd = $targetConn.CreateCommand()
|
|
61
|
+
$deleteCmd.CommandText = "DELETE FROM $fullName;"
|
|
62
|
+
[void]$deleteCmd.ExecuteNonQuery()
|
|
63
|
+
|
|
64
|
+
$selectCmd = $sourceConn.CreateCommand()
|
|
65
|
+
$selectCmd.CommandText = "SELECT * FROM $fullName;"
|
|
66
|
+
|
|
67
|
+
$adapter = New-Object System.Data.SqlClient.SqlDataAdapter $selectCmd
|
|
68
|
+
$dataTable = New-Object System.Data.DataTable
|
|
69
|
+
[void]$adapter.Fill($dataTable)
|
|
70
|
+
|
|
71
|
+
if ($dataTable.Rows.Count -gt 0) {
|
|
72
|
+
$bulkCopy = New-Object System.Data.SqlClient.SqlBulkCopy($targetConn, [System.Data.SqlClient.SqlBulkCopyOptions]::KeepIdentity, $null)
|
|
73
|
+
$bulkCopy.DestinationTableName = $fullName
|
|
74
|
+
$bulkCopy.BulkCopyTimeout = 0
|
|
75
|
+
foreach ($col in $dataTable.Columns) {
|
|
76
|
+
[void]$bulkCopy.ColumnMappings.Add($col.ColumnName, $col.ColumnName)
|
|
77
|
+
}
|
|
78
|
+
$bulkCopy.WriteToServer($dataTable)
|
|
79
|
+
$bulkCopy.Close()
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
$summary.Add([PSCustomObject]@{ Table = $fullName; Rows = $dataTable.Rows.Count })
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
foreach ($t in $tables) {
|
|
86
|
+
$fullName = "[$($t.Schema)].[$($t.Table)]"
|
|
87
|
+
$enableCmd = $targetConn.CreateCommand()
|
|
88
|
+
$enableCmd.CommandText = "ALTER TABLE $fullName WITH CHECK CHECK CONSTRAINT ALL;"
|
|
89
|
+
[void]$enableCmd.ExecuteNonQuery()
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
$summary | Sort-Object Table | Format-Table -AutoSize | Out-String | Write-Output
|
|
93
|
+
Write-Output 'MIGRATION_COMPLETED'
|
|
94
|
+
}
|
|
95
|
+
finally {
|
|
96
|
+
if ($sourceConn.State -eq 'Open') { $sourceConn.Close() }
|
|
97
|
+
if ($targetConn.State -eq 'Open') { $targetConn.Close() }
|
|
98
|
+
}
|
|
@@ -4,6 +4,7 @@ import json
|
|
|
4
4
|
import os
|
|
5
5
|
import sys
|
|
6
6
|
import uuid
|
|
7
|
+
from collections import Counter
|
|
7
8
|
from datetime import datetime, timezone
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from typing import Any
|
|
@@ -25,6 +26,28 @@ def _read_json(path: Path) -> dict[str, Any]:
|
|
|
25
26
|
return payload if isinstance(payload, dict) else {}
|
|
26
27
|
|
|
27
28
|
|
|
29
|
+
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
30
|
+
if not path.exists():
|
|
31
|
+
return []
|
|
32
|
+
rows: list[dict[str, Any]] = []
|
|
33
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
34
|
+
raw = line.strip()
|
|
35
|
+
if not raw:
|
|
36
|
+
continue
|
|
37
|
+
try:
|
|
38
|
+
obj = json.loads(raw)
|
|
39
|
+
except Exception:
|
|
40
|
+
continue
|
|
41
|
+
if isinstance(obj, dict):
|
|
42
|
+
rows.append(obj)
|
|
43
|
+
return rows
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
47
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
49
|
+
|
|
50
|
+
|
|
28
51
|
def _iso_now() -> datetime:
|
|
29
52
|
return datetime.now(timezone.utc)
|
|
30
53
|
|
|
@@ -48,6 +71,161 @@ def _flatten_kpis(payload: dict[str, Any]) -> dict[str, Any]:
|
|
|
48
71
|
return flat
|
|
49
72
|
|
|
50
73
|
|
|
74
|
+
def _build_telemetry_snapshot(repo_root: Path) -> dict[str, Any]:
|
|
75
|
+
traces_path = repo_root / ".telemetry" / "traces.jsonl"
|
|
76
|
+
metrics_path = repo_root / ".telemetry" / "metrics.jsonl"
|
|
77
|
+
traces = _read_jsonl(traces_path)
|
|
78
|
+
metrics = _read_jsonl(metrics_path)
|
|
79
|
+
|
|
80
|
+
execution_ids: set[str] = set()
|
|
81
|
+
operations = Counter()
|
|
82
|
+
sessions = Counter()
|
|
83
|
+
engines = Counter()
|
|
84
|
+
agents = Counter()
|
|
85
|
+
warnings = 0
|
|
86
|
+
errors = 0
|
|
87
|
+
fallback_count = 0
|
|
88
|
+
grounded_count = 0
|
|
89
|
+
routing_count = 0
|
|
90
|
+
durations: list[float] = []
|
|
91
|
+
exporter_failures = Counter()
|
|
92
|
+
|
|
93
|
+
for row in traces:
|
|
94
|
+
context = row.get("context", {}) if isinstance(row.get("context"), dict) else {}
|
|
95
|
+
payload = row.get("payload", {}) if isinstance(row.get("payload"), dict) else {}
|
|
96
|
+
event_name = str(row.get("event_name", ""))
|
|
97
|
+
level = str(row.get("level", "INFO")).upper()
|
|
98
|
+
|
|
99
|
+
execution_id = str(context.get("execution_id", "")).strip()
|
|
100
|
+
if execution_id:
|
|
101
|
+
execution_ids.add(execution_id)
|
|
102
|
+
|
|
103
|
+
session_id = str(context.get("session_id", "")).strip()
|
|
104
|
+
if session_id:
|
|
105
|
+
sessions[session_id] += 1
|
|
106
|
+
|
|
107
|
+
if level == "WARNING":
|
|
108
|
+
warnings += 1
|
|
109
|
+
if level == "ERROR":
|
|
110
|
+
errors += 1
|
|
111
|
+
|
|
112
|
+
if event_name == "ExecutionStarted":
|
|
113
|
+
op = str(payload.get("operation", "")).strip() or "unknown"
|
|
114
|
+
operations[op] += 1
|
|
115
|
+
|
|
116
|
+
if event_name == "RoutingResolved":
|
|
117
|
+
routing_count += 1
|
|
118
|
+
engine = str(payload.get("engine", "")).strip() or "unknown"
|
|
119
|
+
agent = str(payload.get("agent", "")).strip() or "unknown"
|
|
120
|
+
engines[engine] += 1
|
|
121
|
+
agents[agent] += 1
|
|
122
|
+
if bool(payload.get("fallback", False)):
|
|
123
|
+
fallback_count += 1
|
|
124
|
+
if bool(payload.get("grounded", False)):
|
|
125
|
+
grounded_count += 1
|
|
126
|
+
|
|
127
|
+
if event_name == "SpanFinished":
|
|
128
|
+
try:
|
|
129
|
+
durations.append(float(payload.get("duration_ms") or 0.0))
|
|
130
|
+
except Exception:
|
|
131
|
+
pass
|
|
132
|
+
|
|
133
|
+
if event_name == "WarningGenerated" and str(payload.get("warning", "")) == "exporter_failed":
|
|
134
|
+
exporter = str(payload.get("exporter", "unknown")).strip() or "unknown"
|
|
135
|
+
exporter_failures[exporter] += 1
|
|
136
|
+
|
|
137
|
+
metric_totals = {
|
|
138
|
+
"tokens_input": 0.0,
|
|
139
|
+
"tokens_output": 0.0,
|
|
140
|
+
"total_tokens": 0.0,
|
|
141
|
+
"estimated_cost": 0.0,
|
|
142
|
+
"efficiency_score_samples": 0,
|
|
143
|
+
"efficiency_score_avg": 0.0,
|
|
144
|
+
}
|
|
145
|
+
efficiency_sum = 0.0
|
|
146
|
+
for row in metrics:
|
|
147
|
+
payload = row.get("payload", {}) if isinstance(row.get("payload"), dict) else {}
|
|
148
|
+
if str(row.get("event_name", "")) != "MetricCalculated":
|
|
149
|
+
continue
|
|
150
|
+
name = str(payload.get("name", "")).strip()
|
|
151
|
+
try:
|
|
152
|
+
value = float(payload.get("value") or 0.0)
|
|
153
|
+
except Exception:
|
|
154
|
+
continue
|
|
155
|
+
|
|
156
|
+
if name in {"tokens_input", "tokens_output", "total_tokens", "estimated_cost"}:
|
|
157
|
+
metric_totals[name] += value
|
|
158
|
+
elif name == "efficiency_score":
|
|
159
|
+
efficiency_sum += value
|
|
160
|
+
metric_totals["efficiency_score_samples"] += 1
|
|
161
|
+
|
|
162
|
+
samples = int(metric_totals["efficiency_score_samples"])
|
|
163
|
+
if samples > 0:
|
|
164
|
+
metric_totals["efficiency_score_avg"] = round(efficiency_sum / samples, 4)
|
|
165
|
+
|
|
166
|
+
avg_duration = round(sum(durations) / len(durations), 4) if durations else 0.0
|
|
167
|
+
p95_duration = 0.0
|
|
168
|
+
if durations:
|
|
169
|
+
sorted_d = sorted(durations)
|
|
170
|
+
idx = max(0, min(len(sorted_d) - 1, int(round(0.95 * (len(sorted_d) - 1)))))
|
|
171
|
+
p95_duration = round(sorted_d[idx], 4)
|
|
172
|
+
|
|
173
|
+
fallback_rate = round((fallback_count / routing_count), 4) if routing_count else 0.0
|
|
174
|
+
grounded_rate = round((grounded_count / routing_count), 4) if routing_count else 0.0
|
|
175
|
+
|
|
176
|
+
return {
|
|
177
|
+
"timestamp": _iso_now().isoformat(),
|
|
178
|
+
"source": {
|
|
179
|
+
"traces": str(traces_path),
|
|
180
|
+
"metrics": str(metrics_path),
|
|
181
|
+
},
|
|
182
|
+
"totals": {
|
|
183
|
+
"trace_records": len(traces),
|
|
184
|
+
"metric_records": len(metrics),
|
|
185
|
+
"executions": len(execution_ids),
|
|
186
|
+
"warnings": warnings,
|
|
187
|
+
"errors": errors,
|
|
188
|
+
},
|
|
189
|
+
"kpis": {
|
|
190
|
+
"routing_fallback_rate": fallback_rate,
|
|
191
|
+
"routing_grounded_rate": grounded_rate,
|
|
192
|
+
"avg_span_duration_ms": avg_duration,
|
|
193
|
+
"p95_span_duration_ms": p95_duration,
|
|
194
|
+
"tokens_input": int(metric_totals["tokens_input"]),
|
|
195
|
+
"tokens_output": int(metric_totals["tokens_output"]),
|
|
196
|
+
"total_tokens": int(metric_totals["total_tokens"]),
|
|
197
|
+
"total_cost_usd": round(float(metric_totals["estimated_cost"]), 6),
|
|
198
|
+
"efficiency_score_avg": metric_totals["efficiency_score_avg"],
|
|
199
|
+
},
|
|
200
|
+
"breakdown": {
|
|
201
|
+
"operation": dict(operations),
|
|
202
|
+
"session": dict(sessions),
|
|
203
|
+
"engine": dict(engines),
|
|
204
|
+
"agent": dict(agents),
|
|
205
|
+
"exporter_failures": dict(exporter_failures),
|
|
206
|
+
},
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _build_chat_usage_snapshot(repo_root: Path) -> dict[str, Any]:
|
|
211
|
+
chat_path = repo_root / "observability" / "evals" / "chat-token-usage-report.json"
|
|
212
|
+
payload = _read_json(chat_path)
|
|
213
|
+
totals = payload.get("totals", {}) if isinstance(payload.get("totals", {}), dict) else {}
|
|
214
|
+
budget = payload.get("budget", {}) if isinstance(payload.get("budget", {}), dict) else {}
|
|
215
|
+
return {
|
|
216
|
+
"timestamp": payload.get("timestamp"),
|
|
217
|
+
"source": str(chat_path),
|
|
218
|
+
"kpis": {
|
|
219
|
+
"input_tokens": int(totals.get("input_tokens", 0) or 0),
|
|
220
|
+
"output_tokens": int(totals.get("output_tokens", 0) or 0),
|
|
221
|
+
"total_tokens": int(totals.get("total_tokens", 0) or 0),
|
|
222
|
+
"copilot_credits": float(totals.get("copilot_credits", 0.0) or 0.0),
|
|
223
|
+
"budget_utilization_rate": float(budget.get("budget_utilization_rate", 0.0) or 0.0),
|
|
224
|
+
},
|
|
225
|
+
"raw": payload,
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
|
|
51
229
|
def _publish_run(
|
|
52
230
|
client: Any,
|
|
53
231
|
*,
|
|
@@ -104,12 +282,18 @@ def _publish_run(
|
|
|
104
282
|
|
|
105
283
|
|
|
106
284
|
def _publish_feedback_metrics(client: Any, *, run_id: str, metrics: dict[str, Any], source: str) -> None:
|
|
285
|
+
min_score = -99999.9999
|
|
286
|
+
max_score = 99999.9999
|
|
107
287
|
for key, raw_value in metrics.items():
|
|
108
288
|
try:
|
|
109
289
|
value = float(raw_value)
|
|
110
290
|
except Exception:
|
|
111
291
|
continue
|
|
112
292
|
|
|
293
|
+
if value < min_score or value > max_score:
|
|
294
|
+
# LangSmith feedback score has bounded range.
|
|
295
|
+
continue
|
|
296
|
+
|
|
113
297
|
try:
|
|
114
298
|
client.create_feedback(
|
|
115
299
|
run_id=run_id,
|
|
@@ -152,9 +336,38 @@ def main() -> int:
|
|
|
152
336
|
|
|
153
337
|
learning_path = repo_root / "observability" / "evals" / "learning-loop-report.json"
|
|
154
338
|
value_path = repo_root / "observability" / "evals" / "iteration-value-report.json"
|
|
339
|
+
telemetry_snapshot_path = repo_root / "observability" / "evals" / "telemetry-flow-cost-token-report.json"
|
|
340
|
+
chat_snapshot_path = repo_root / "observability" / "evals" / "chat-token-usage-report.json"
|
|
155
341
|
|
|
156
342
|
learning = _read_json(learning_path)
|
|
157
343
|
value = _read_json(value_path)
|
|
344
|
+
telemetry_snapshot = _build_telemetry_snapshot(repo_root)
|
|
345
|
+
_write_json(telemetry_snapshot_path, telemetry_snapshot)
|
|
346
|
+
chat_snapshot = _build_chat_usage_snapshot(repo_root)
|
|
347
|
+
|
|
348
|
+
project_traceability_path = (
|
|
349
|
+
repo_root
|
|
350
|
+
/ "projects"
|
|
351
|
+
/ "techriders"
|
|
352
|
+
/ "analysis_mcpee"
|
|
353
|
+
/ f"telemetry-recovery-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.json"
|
|
354
|
+
)
|
|
355
|
+
unified_snapshot = {
|
|
356
|
+
"timestamp": _iso_now().isoformat(),
|
|
357
|
+
"host_project": host_project,
|
|
358
|
+
"telemetry_scope": telemetry_scope,
|
|
359
|
+
"sources": {
|
|
360
|
+
"learning": str(learning_path),
|
|
361
|
+
"iteration_value": str(value_path),
|
|
362
|
+
"telemetry_snapshot": str(telemetry_snapshot_path),
|
|
363
|
+
"chat_usage": str(chat_snapshot_path),
|
|
364
|
+
},
|
|
365
|
+
"learning": learning,
|
|
366
|
+
"iteration_value": value,
|
|
367
|
+
"telemetry": telemetry_snapshot,
|
|
368
|
+
"chat_usage": chat_snapshot,
|
|
369
|
+
}
|
|
370
|
+
_write_json(project_traceability_path, unified_snapshot)
|
|
158
371
|
|
|
159
372
|
published = 0
|
|
160
373
|
|
|
@@ -288,10 +501,92 @@ def main() -> int:
|
|
|
288
501
|
_publish_feedback_metrics(client, run_id=run_id, metrics=alignment_feedback, source="alignment")
|
|
289
502
|
published += 1
|
|
290
503
|
|
|
504
|
+
telemetry_kpis = telemetry_snapshot.get("kpis", {}) if isinstance(telemetry_snapshot.get("kpis", {}), dict) else {}
|
|
505
|
+
if telemetry_kpis:
|
|
506
|
+
telemetry_flat_kpis = _flatten_kpis(telemetry_kpis)
|
|
507
|
+
run_id = _publish_run(
|
|
508
|
+
client,
|
|
509
|
+
project=cfg.langsmith.project,
|
|
510
|
+
name="KPI::TelemetryFlowCostToken",
|
|
511
|
+
inputs={
|
|
512
|
+
"source_file": str(telemetry_snapshot_path),
|
|
513
|
+
"timestamp": telemetry_snapshot.get("timestamp"),
|
|
514
|
+
},
|
|
515
|
+
outputs={
|
|
516
|
+
"kpis": telemetry_kpis,
|
|
517
|
+
**telemetry_flat_kpis,
|
|
518
|
+
"totals": telemetry_snapshot.get("totals", {}),
|
|
519
|
+
"breakdown": telemetry_snapshot.get("breakdown", {}),
|
|
520
|
+
},
|
|
521
|
+
tags=[
|
|
522
|
+
"mcpee",
|
|
523
|
+
"kpi",
|
|
524
|
+
"telemetry",
|
|
525
|
+
"cost-token-flow",
|
|
526
|
+
"dashboard",
|
|
527
|
+
f"scope:{telemetry_scope}",
|
|
528
|
+
f"host:{host_slug}",
|
|
529
|
+
],
|
|
530
|
+
metadata={
|
|
531
|
+
"kpi_source": "telemetry-flow-cost-token",
|
|
532
|
+
"kpi_kind": "snapshot",
|
|
533
|
+
"host_project": host_project,
|
|
534
|
+
"host_project_slug": host_slug,
|
|
535
|
+
"telemetry_scope": telemetry_scope,
|
|
536
|
+
**telemetry_flat_kpis,
|
|
537
|
+
},
|
|
538
|
+
total_cost=float(telemetry_kpis.get("total_cost_usd", 0.0) or 0.0),
|
|
539
|
+
total_tokens=int(telemetry_kpis.get("total_tokens", 0) or 0),
|
|
540
|
+
run_type="llm",
|
|
541
|
+
)
|
|
542
|
+
_publish_feedback_metrics(client, run_id=run_id, metrics=telemetry_flat_kpis, source="telemetry-flow-cost-token")
|
|
543
|
+
published += 1
|
|
544
|
+
|
|
545
|
+
chat_kpis = chat_snapshot.get("kpis", {}) if isinstance(chat_snapshot.get("kpis", {}), dict) else {}
|
|
546
|
+
if chat_kpis:
|
|
547
|
+
chat_flat_kpis = _flatten_kpis(chat_kpis)
|
|
548
|
+
run_id = _publish_run(
|
|
549
|
+
client,
|
|
550
|
+
project=cfg.langsmith.project,
|
|
551
|
+
name="KPI::ChatTokenUsage",
|
|
552
|
+
inputs={
|
|
553
|
+
"source_file": str(chat_snapshot_path),
|
|
554
|
+
"timestamp": chat_snapshot.get("timestamp"),
|
|
555
|
+
},
|
|
556
|
+
outputs={
|
|
557
|
+
"kpis": chat_kpis,
|
|
558
|
+
**chat_flat_kpis,
|
|
559
|
+
"budget": (chat_snapshot.get("raw", {}) or {}).get("budget", {}),
|
|
560
|
+
},
|
|
561
|
+
tags=[
|
|
562
|
+
"mcpee",
|
|
563
|
+
"kpi",
|
|
564
|
+
"chat-usage",
|
|
565
|
+
"cost-token-flow",
|
|
566
|
+
"dashboard",
|
|
567
|
+
f"scope:{telemetry_scope}",
|
|
568
|
+
f"host:{host_slug}",
|
|
569
|
+
],
|
|
570
|
+
metadata={
|
|
571
|
+
"kpi_source": "chat-token-usage",
|
|
572
|
+
"kpi_kind": "snapshot",
|
|
573
|
+
"host_project": host_project,
|
|
574
|
+
"host_project_slug": host_slug,
|
|
575
|
+
"telemetry_scope": telemetry_scope,
|
|
576
|
+
**chat_flat_kpis,
|
|
577
|
+
},
|
|
578
|
+
total_tokens=int(chat_kpis.get("total_tokens", 0) or 0),
|
|
579
|
+
run_type="llm",
|
|
580
|
+
)
|
|
581
|
+
_publish_feedback_metrics(client, run_id=run_id, metrics=chat_flat_kpis, source="chat-token-usage")
|
|
582
|
+
published += 1
|
|
583
|
+
|
|
291
584
|
print(
|
|
292
585
|
f"Published KPI runs: {published} to project '{cfg.langsmith.project}' "
|
|
293
586
|
f"(host_project='{host_project}', scope='{telemetry_scope}')."
|
|
294
587
|
)
|
|
588
|
+
print(f"Local telemetry snapshot: {telemetry_snapshot_path}")
|
|
589
|
+
print(f"Project traceability snapshot: {project_traceability_path}")
|
|
295
590
|
if published == 0:
|
|
296
591
|
print("No local KPI report JSON files were found in observability/evals.")
|
|
297
592
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
param(
|
|
2
|
+
[switch]$DryRun
|
|
3
|
+
)
|
|
4
|
+
|
|
5
|
+
$ErrorActionPreference = 'Stop'
|
|
6
|
+
|
|
7
|
+
function Resolve-RepoRoot {
|
|
8
|
+
return (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function Has-CodegraphIndex([string]$path) {
|
|
12
|
+
return (Test-Path (Join-Path $path '.codegraph'))
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function Resolve-TargetPath {
|
|
16
|
+
param([string]$RepoRoot)
|
|
17
|
+
|
|
18
|
+
$envPath = [Environment]::GetEnvironmentVariable('CODEGRAPH_PROJECT_PATH')
|
|
19
|
+
if (-not [string]::IsNullOrWhiteSpace($envPath)) {
|
|
20
|
+
$resolvedEnvPath = $null
|
|
21
|
+
try {
|
|
22
|
+
$resolvedEnvPath = (Resolve-Path $envPath).Path
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
$resolvedEnvPath = $null
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if ($resolvedEnvPath -and (Has-CodegraphIndex $resolvedEnvPath)) {
|
|
29
|
+
return $resolvedEnvPath
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
$projectCandidates = @()
|
|
34
|
+
$projectsRoot = Join-Path $RepoRoot 'projects'
|
|
35
|
+
if (Test-Path $projectsRoot) {
|
|
36
|
+
foreach ($projectDir in Get-ChildItem -Path $projectsRoot -Directory -ErrorAction SilentlyContinue) {
|
|
37
|
+
if (Has-CodegraphIndex $projectDir.FullName) {
|
|
38
|
+
$projectCandidates += $projectDir.FullName
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if ($projectCandidates.Count -eq 1) {
|
|
44
|
+
return $projectCandidates[0]
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (Has-CodegraphIndex $RepoRoot) {
|
|
48
|
+
return $RepoRoot
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if ($projectCandidates.Count -gt 1) {
|
|
52
|
+
return $projectCandidates[0]
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return $RepoRoot
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
$repoRoot = Resolve-RepoRoot
|
|
59
|
+
$targetPath = Resolve-TargetPath -RepoRoot $repoRoot
|
|
60
|
+
|
|
61
|
+
if ($DryRun) {
|
|
62
|
+
Write-Output $targetPath
|
|
63
|
+
exit 0
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
& codegraph serve --mcp --path $targetPath
|
|
67
|
+
exit $LASTEXITCODE
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
param(
|
|
2
|
+
[switch]$DryRun
|
|
3
|
+
)
|
|
4
|
+
|
|
5
|
+
$ErrorActionPreference = 'Stop'
|
|
6
|
+
|
|
7
|
+
function Resolve-RepoRoot {
|
|
8
|
+
return (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function Is-GitRepo([string]$path) {
|
|
12
|
+
return (Test-Path (Join-Path $path '.git'))
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function Has-GitNexusIndex([string]$path) {
|
|
16
|
+
return (Test-Path (Join-Path $path '.gitnexus'))
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function Resolve-TargetPath {
|
|
20
|
+
param([string]$RepoRoot)
|
|
21
|
+
|
|
22
|
+
$envPath = [Environment]::GetEnvironmentVariable('GITNEXUS_PROJECT_PATH')
|
|
23
|
+
if (-not [string]::IsNullOrWhiteSpace($envPath)) {
|
|
24
|
+
$resolvedEnvPath = $null
|
|
25
|
+
try {
|
|
26
|
+
$resolvedEnvPath = (Resolve-Path $envPath).Path
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
$resolvedEnvPath = $null
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if ($resolvedEnvPath -and (Is-GitRepo $resolvedEnvPath)) {
|
|
33
|
+
return $resolvedEnvPath
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
$projectCandidates = @()
|
|
38
|
+
$indexedProjectCandidates = @()
|
|
39
|
+
$projectsRoot = Join-Path $RepoRoot 'projects'
|
|
40
|
+
|
|
41
|
+
if (Test-Path $projectsRoot) {
|
|
42
|
+
foreach ($projectDir in Get-ChildItem -Path $projectsRoot -Directory -ErrorAction SilentlyContinue) {
|
|
43
|
+
if (Is-GitRepo $projectDir.FullName) {
|
|
44
|
+
$projectCandidates += $projectDir.FullName
|
|
45
|
+
if (Has-GitNexusIndex $projectDir.FullName) {
|
|
46
|
+
$indexedProjectCandidates += $projectDir.FullName
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
if ($indexedProjectCandidates.Count -eq 1) {
|
|
53
|
+
return $indexedProjectCandidates[0]
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if ((Has-GitNexusIndex $RepoRoot) -and (Is-GitRepo $RepoRoot)) {
|
|
57
|
+
return $RepoRoot
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if ($indexedProjectCandidates.Count -gt 1) {
|
|
61
|
+
return $indexedProjectCandidates[0]
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if ($projectCandidates.Count -eq 1) {
|
|
65
|
+
return $projectCandidates[0]
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if ($projectCandidates.Count -gt 1) {
|
|
69
|
+
return $projectCandidates[0]
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return $RepoRoot
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
$repoRoot = Resolve-RepoRoot
|
|
76
|
+
$targetPath = Resolve-TargetPath -RepoRoot $repoRoot
|
|
77
|
+
|
|
78
|
+
if ($DryRun) {
|
|
79
|
+
Write-Output $targetPath
|
|
80
|
+
exit 0
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
Set-Location $targetPath
|
|
84
|
+
& gitnexus mcp
|
|
85
|
+
exit $LASTEXITCODE
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _bootstrap_repo_root() -> Path:
|
|
10
|
+
repo_root = Path(__file__).resolve().parents[2]
|
|
11
|
+
if str(repo_root) not in sys.path:
|
|
12
|
+
sys.path.insert(0, str(repo_root))
|
|
13
|
+
return repo_root
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _parse_args() -> argparse.Namespace:
|
|
17
|
+
parser = argparse.ArgumentParser(description="Run a command wrapped with telemetry execution/span.")
|
|
18
|
+
parser.add_argument("--operation", required=True, help="Telemetry operation name.")
|
|
19
|
+
parser.add_argument("--session-id", default="mcpee-cli", help="Telemetry session identifier.")
|
|
20
|
+
parser.add_argument("--cwd", default=".", help="Working directory for the command.")
|
|
21
|
+
parser.add_argument("command", nargs=argparse.REMAINDER, help="Command to execute after --.")
|
|
22
|
+
return parser.parse_args()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _normalize_command(raw: list[str]) -> list[str]:
|
|
26
|
+
cmd = list(raw)
|
|
27
|
+
if cmd and cmd[0] == "--":
|
|
28
|
+
cmd = cmd[1:]
|
|
29
|
+
return cmd
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def main() -> int:
|
|
33
|
+
args = _parse_args()
|
|
34
|
+
command = _normalize_command(args.command)
|
|
35
|
+
if not command:
|
|
36
|
+
print("trace-command.py: missing command to execute", file=sys.stderr)
|
|
37
|
+
return 2
|
|
38
|
+
|
|
39
|
+
repo_root = _bootstrap_repo_root()
|
|
40
|
+
from telemetry import build_telemetry_collector
|
|
41
|
+
|
|
42
|
+
collector = build_telemetry_collector(repo_root)
|
|
43
|
+
cwd = Path(args.cwd).resolve()
|
|
44
|
+
exit_code = 0
|
|
45
|
+
|
|
46
|
+
try:
|
|
47
|
+
with collector.start_execution(operation=args.operation, session_id=args.session_id, model="mcpee-cli"):
|
|
48
|
+
with collector.start_span(
|
|
49
|
+
name="subprocess",
|
|
50
|
+
kind="TOOL",
|
|
51
|
+
attributes={
|
|
52
|
+
"cwd": str(cwd),
|
|
53
|
+
"command": command[0],
|
|
54
|
+
"args_count": len(command) - 1,
|
|
55
|
+
},
|
|
56
|
+
):
|
|
57
|
+
completed = subprocess.run(command, cwd=str(cwd), check=False)
|
|
58
|
+
exit_code = int(completed.returncode)
|
|
59
|
+
collector.record_metric("subprocess_exit_code", float(exit_code), unit="code")
|
|
60
|
+
if exit_code != 0:
|
|
61
|
+
raise RuntimeError(f"command failed with exit code {exit_code}")
|
|
62
|
+
except RuntimeError:
|
|
63
|
+
pass
|
|
64
|
+
|
|
65
|
+
return exit_code
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
if __name__ == "__main__":
|
|
69
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
param(
|
|
2
|
+
[switch]$AllHosts,
|
|
3
|
+
[switch]$CurrentProcess
|
|
4
|
+
)
|
|
5
|
+
|
|
6
|
+
Set-StrictMode -Version Latest
|
|
7
|
+
$ErrorActionPreference = 'Stop'
|
|
8
|
+
|
|
9
|
+
function Remove-FromProfile {
|
|
10
|
+
param([string]$ProfilePath)
|
|
11
|
+
|
|
12
|
+
if (-not (Test-Path $ProfilePath)) {
|
|
13
|
+
return
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
$content = Get-Content -Path $ProfilePath -Raw -ErrorAction SilentlyContinue
|
|
17
|
+
if ($null -eq $content) {
|
|
18
|
+
return
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if ($content -match 'MCPEE_TERMINAL_TELEMETRY_START') {
|
|
22
|
+
$updated = [regex]::Replace(
|
|
23
|
+
$content,
|
|
24
|
+
'(?s)\r?\n?# >>> MCPEE_TERMINAL_TELEMETRY_START >>>.*?# <<< MCPEE_TERMINAL_TELEMETRY_END <<<\r?\n?',
|
|
25
|
+
"`r`n"
|
|
26
|
+
)
|
|
27
|
+
Set-Content -Path $ProfilePath -Value $updated -Encoding UTF8
|
|
28
|
+
Write-Host "[ok] Hook eliminado de: $ProfilePath"
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if ($CurrentProcess) {
|
|
33
|
+
Remove-Item Function:\global:prompt -ErrorAction SilentlyContinue
|
|
34
|
+
if (Test-Path Function:\global:MCPEE_ORIGINAL_PROMPT) {
|
|
35
|
+
Set-Item -Path Function:\global:prompt -Value ((Get-Command MCPEE_ORIGINAL_PROMPT -CommandType Function).ScriptBlock) -Force
|
|
36
|
+
Remove-Item Function:\global:MCPEE_ORIGINAL_PROMPT -ErrorAction SilentlyContinue
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
Remove-Item Function:\global:Invoke-McpeeTerminalTelemetry -ErrorAction SilentlyContinue
|
|
40
|
+
Remove-Item Function:\global:Get-McpeeTelemetryPythonCommand -ErrorAction SilentlyContinue
|
|
41
|
+
Remove-Variable MCPEE_TELEMETRY_REPO_ROOT -Scope Global -ErrorAction SilentlyContinue
|
|
42
|
+
Remove-Variable MCPEE_TELEMETRY_SESSION_ID -Scope Global -ErrorAction SilentlyContinue
|
|
43
|
+
Remove-Variable MCPEE_TELEMETRY_LAST_HISTORY_ID -Scope Global -ErrorAction SilentlyContinue
|
|
44
|
+
Remove-Variable MCPEE_TELEMETRY_LAST_STAMP -Scope Global -ErrorAction SilentlyContinue
|
|
45
|
+
Write-Host '[ok] Hook desactivado en la sesion actual.'
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
Remove-FromProfile -ProfilePath $PROFILE.CurrentUserCurrentHost
|
|
49
|
+
if ($AllHosts) {
|
|
50
|
+
Remove-FromProfile -ProfilePath $PROFILE.CurrentUserAllHosts
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
Write-Host '[ok] Telemetria global de terminal desactivada.'
|