@inspectr/mcplab 1.7.0 โ 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/dist/app/assets/index-BLsLqeX3.js +249 -0
- package/dist/app/assets/index-D80zV1LL.css +1 -0
- package/dist/app/icons/claude-color.svg +1 -0
- package/dist/app/icons/openai.svg +1 -0
- package/dist/app/index.html +2 -2
- package/dist/app-server/app-context.d.ts +129 -75
- package/dist/app-server/assistant-common.d.ts +2 -0
- package/dist/app-server/assistant-common.d.ts.map +1 -1
- package/dist/app-server/assistant-common.js +2 -1
- package/dist/app-server/assistant-common.js.map +1 -1
- package/dist/app-server/config-store.d.ts +2 -0
- package/dist/app-server/config-store.d.ts.map +1 -1
- package/dist/app-server/config-store.js +10 -5
- package/dist/app-server/config-store.js.map +1 -1
- package/dist/app-server/oauth-debugger-domain.d.ts +202 -174
- package/dist/app-server/oauth-debugger-domain.d.ts.map +1 -1
- package/dist/app-server/oauth-debugger-domain.js +53 -4
- package/dist/app-server/oauth-debugger-domain.js.map +1 -1
- package/dist/app-server/oauth-runtime-domain.d.ts +53 -42
- package/dist/app-server/oauth-runtime-domain.d.ts.map +1 -1
- package/dist/app-server/oauth-runtime-domain.js +1 -3
- package/dist/app-server/oauth-runtime-domain.js.map +1 -1
- package/dist/app-server/oauth-runtime-routes.d.ts +13 -11
- package/dist/app-server/oauth-runtime-routes.d.ts.map +1 -1
- package/dist/app-server/oauth-runtime-routes.js +27 -1
- package/dist/app-server/oauth-runtime-routes.js.map +1 -1
- package/dist/app-server/oauth-session-manager.d.ts +57 -0
- package/dist/app-server/oauth-session-manager.d.ts.map +1 -0
- package/dist/app-server/oauth-session-manager.js +338 -0
- package/dist/app-server/oauth-session-manager.js.map +1 -0
- package/dist/app-server/result-assistant-domain.d.ts +5 -3
- package/dist/app-server/result-assistant-domain.d.ts.map +1 -1
- package/dist/app-server/result-assistant-domain.js +38 -44
- package/dist/app-server/result-assistant-domain.js.map +1 -1
- package/dist/app-server/result-assistant-tools.d.ts +11 -0
- package/dist/app-server/result-assistant-tools.d.ts.map +1 -0
- package/dist/app-server/result-assistant-tools.js +27 -0
- package/dist/app-server/result-assistant-tools.js.map +1 -0
- package/dist/app-server/result-assistant.d.ts.map +1 -1
- package/dist/app-server/result-assistant.js +24 -27
- package/dist/app-server/result-assistant.js.map +1 -1
- package/dist/app-server/router.d.ts.map +1 -1
- package/dist/app-server/router.js +36 -8
- package/dist/app-server/router.js.map +1 -1
- package/dist/app-server/runs-routes.d.ts +54 -30
- package/dist/app-server/runs-routes.d.ts.map +1 -1
- package/dist/app-server/runs-routes.js +207 -87
- package/dist/app-server/runs-routes.js.map +1 -1
- package/dist/app-server/scenario-assistant-domain.d.ts +19 -3
- package/dist/app-server/scenario-assistant-domain.d.ts.map +1 -1
- package/dist/app-server/scenario-assistant-domain.js +272 -6
- package/dist/app-server/scenario-assistant-domain.js.map +1 -1
- package/dist/app-server/scenario-assistant.d.ts +2 -0
- package/dist/app-server/scenario-assistant.d.ts.map +1 -1
- package/dist/app-server/scenario-assistant.js +60 -3
- package/dist/app-server/scenario-assistant.js.map +1 -1
- package/dist/app-server/tool-analysis-domain.d.ts +179 -165
- package/dist/app-server/tool-analysis-domain.d.ts.map +1 -1
- package/dist/app-server/tool-analysis-domain.js +21 -9
- package/dist/app-server/tool-analysis-domain.js.map +1 -1
- package/dist/app-server/tool-analysis.d.ts +21 -12
- package/dist/app-server/tool-analysis.d.ts.map +1 -1
- package/dist/app-server/tool-analysis.js +23 -20
- package/dist/app-server/tool-analysis.js.map +1 -1
- package/dist/app-server/types.d.ts +7 -0
- package/dist/app-server/types.d.ts.map +1 -1
- package/dist/cli.js +186 -103
- package/dist/cli.js.map +1 -1
- package/dist/eval-config-files.d.ts +13 -0
- package/dist/eval-config-files.d.ts.map +1 -0
- package/dist/eval-config-files.js +89 -0
- package/dist/eval-config-files.js.map +1 -0
- package/dist/run-interactive.d.ts.map +1 -1
- package/dist/run-interactive.js +14 -10
- package/dist/run-interactive.js.map +1 -1
- package/package.json +4 -4
- package/dist/app/assets/index-CYs1291A.css +0 -1
- package/dist/app/assets/index-Fe4npghI.js +0 -249
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
|
-
import {
|
|
2
|
+
import { OAuthAuthorizationRequiredError } from './oauth-session-manager.js';
|
|
3
3
|
import { createToolAnalysisReportId, deleteToolAnalysisReportRecordFromDirs, listToolAnalysisReportsFromDirs, readToolAnalysisReportRecordFromDirs, writeToolAnalysisReportRecord } from './tool-analysis-storage.js';
|
|
4
4
|
function defaultNewToolAnalysisResultsDir(workspaceRoot) {
|
|
5
5
|
return resolve(workspaceRoot, 'mcplab/results/tool-analysis');
|
|
@@ -16,7 +16,7 @@ function resolveToolAnalysisReadDirs(settings) {
|
|
|
16
16
|
return dirs;
|
|
17
17
|
}
|
|
18
18
|
export async function handleToolAnalysisRoutes(params) {
|
|
19
|
-
const { req, res, pathname, method, settings, toolAnalysisJobs,
|
|
19
|
+
const { req, res, pathname, method, settings, toolAnalysisJobs, oauthSessionManager, deps } = params;
|
|
20
20
|
const { parseBody, asJson, addJobEvent, sendSseEvent, readLibraries, discoverMcpToolsForServers, runToolAnalysisJob } = deps;
|
|
21
21
|
const errorMessage = (error) => (error instanceof Error ? error.message : String(error));
|
|
22
22
|
if (pathname === '/api/tool-analysis/discover-tools' && method === 'POST') {
|
|
@@ -29,24 +29,31 @@ export async function handleToolAnalysisRoutes(params) {
|
|
|
29
29
|
return true;
|
|
30
30
|
}
|
|
31
31
|
const libraries = readLibraries(settings.librariesDir);
|
|
32
|
-
const oauthRuntimeSessionsByServer = body.oauthRuntimeSessions && typeof body.oauthRuntimeSessions === 'object'
|
|
33
|
-
? body.oauthRuntimeSessions
|
|
34
|
-
: undefined;
|
|
35
32
|
const oauthServers = serverNames.filter((serverName) => libraries.servers[serverName]?.auth?.type === 'oauth_authorization_code');
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
33
|
+
let serverAuthHeaders;
|
|
34
|
+
try {
|
|
35
|
+
serverAuthHeaders =
|
|
36
|
+
oauthServers.length > 0
|
|
37
|
+
? await oauthSessionManager.getAuthHeadersForServers(oauthServers, req.headers.host)
|
|
38
|
+
: undefined;
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
if (error instanceof OAuthAuthorizationRequiredError) {
|
|
42
|
+
asJson(res, 401, { error: error.message, oauth: { required: error.details } });
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
const { mcp, servers } = await discoverMcpToolsForServers(libraries.servers, serverNames, {
|
|
45
48
|
serverAuthHeaders
|
|
46
49
|
});
|
|
50
|
+
const mcpServerVersions = mcp.getServerVersions();
|
|
51
|
+
const mcpServerImplementations = mcp.getServerImplementations();
|
|
47
52
|
asJson(res, 200, {
|
|
48
53
|
servers: servers.map((entry) => ({
|
|
49
54
|
serverName: entry.serverName,
|
|
55
|
+
mcpServerVersion: mcpServerVersions[entry.serverName] ?? null,
|
|
56
|
+
mcpServerImplementation: mcpServerImplementations[entry.serverName] ?? null,
|
|
50
57
|
warnings: entry.warnings,
|
|
51
58
|
tools: entry.tools.map((tool) => ({
|
|
52
59
|
name: tool.tool.name,
|
|
@@ -81,9 +88,6 @@ export async function handleToolAnalysisRoutes(params) {
|
|
|
81
88
|
const selectedToolsByServer = body.selectedToolsByServer && typeof body.selectedToolsByServer === 'object'
|
|
82
89
|
? body.selectedToolsByServer
|
|
83
90
|
: undefined;
|
|
84
|
-
const oauthRuntimeSessionsByServer = body.oauthRuntimeSessions && typeof body.oauthRuntimeSessions === 'object'
|
|
85
|
-
? body.oauthRuntimeSessions
|
|
86
|
-
: undefined;
|
|
87
91
|
const deeperOptions = body.deeperAnalysisOptions ?? {};
|
|
88
92
|
const maxParallelTools = Math.max(1, Math.min(8, Number(body.maxParallelTools ?? deeperOptions.maxParallelTools ?? 2) || 2));
|
|
89
93
|
const jobId = `ta-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
@@ -114,9 +118,8 @@ export async function handleToolAnalysisRoutes(params) {
|
|
|
114
118
|
? String(body.assistantAgentName)
|
|
115
119
|
: undefined,
|
|
116
120
|
serverNames,
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
oauthDebuggerSessions,
|
|
121
|
+
oauthSessionManager,
|
|
122
|
+
hostHeader: req.headers.host,
|
|
120
123
|
selectedToolsByServer,
|
|
121
124
|
modes,
|
|
122
125
|
deeper: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-analysis.js","sourceRoot":"","sources":["../../src/app-server/tool-analysis.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,OAAO,EAAE,8BAA8B,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EACL,0BAA0B,EAE1B,sCAAsC,EAEtC,+BAA+B,EAE/B,oCAAoC,EACpC,6BAA6B,EAC9B,MAAM,4BAA4B,CAAC;AAapC,SAAS,gCAAgC,CAAC,aAAqB;IAC7D,OAAO,OAAO,CAAC,aAAa,EAAE,8BAA8B,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,mCAAmC,CAAC,aAAqB;IAChE,OAAO,OAAO,CAAC,aAAa,EAAE,8BAA8B,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,2BAA2B,CAAC,QAA4C;IAC/E,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,gCAAgC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,mCAAmC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3E,IAAI,QAAQ,CAAC,sBAAsB,KAAK,WAAW,IAAI,MAAM,KAAK,WAAW;QAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjG,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,MAU9C;IACC,MAAM,EACJ,GAAG,EACH,GAAG,EACH,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,IAAI,EACL,GAAG,MAAM,CAAC;IACX,MAAM,EACJ,SAAS,EACT,MAAM,EACN,WAAW,EACX,YAAY,EACZ,aAAa,EACb,0BAA0B,EAC1B,kBAAkB,EACnB,GAAG,IAAI,CAAC;IAET,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,EAAE,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAElG,IAAI,QAAQ,KAAK,mCAAmC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC1E,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QACP,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,iDAAiD,EAAE,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,4BAA4B,GAChC,IAAI,CAAC,oBAAoB,IAAI,OAAO,IAAI,CAAC,oBAAoB,KAAK,QAAQ;YACxE,CAAC,CAAE,IAAI,CAAC,oBAA+C;YACvD,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CACrC,CAAC,UAAkB,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,KAAK,0BAA0B,CACjG,CAAC;QACF,MAAM,iBAAiB,GACrB,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,8BAA8B,CAAC;gBAC7B,mBAAmB,EAAE,YAAY;gBACjC,4BAA4B;gBAC5B,eAAe,EAAE,oBAAoB;gBACrC,qBAAqB;aACtB,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,0BAA0B,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,EAAE;YACnF,iBAAiB;SAClB,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;YACf,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC/B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAChC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;oBACpB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;oBACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;oBAClC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;oBAClC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY;oBACpC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;iBAChD,CAAC,CAAC;aACJ,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,KAAK,yBAAyB,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,KAAK,GAAG;YACZ,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,cAAc,CAAC;YACpD,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,cAAc,CAAC;SACrD,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,iDAAiD,EAAE,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;YACnD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,mCAAmC,EAAE,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,qBAAqB,GACzB,IAAI,CAAC,qBAAqB,IAAI,OAAO,IAAI,CAAC,qBAAqB,KAAK,QAAQ;YAC1E,CAAC,CAAE,IAAI,CAAC,qBAAkD;YAC1D,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,4BAA4B,GAChC,IAAI,CAAC,oBAAoB,IAAI,OAAO,IAAI,CAAC,oBAAoB,KAAK,QAAQ;YACxE,CAAC,CAAE,IAAI,CAAC,oBAA+C;YACvD,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,IAAI,EAAE,CAAC;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CACvF,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,GAAG,GAAoB;YAC3B,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,IAAI,GAAG,EAAE;YAClB,eAAe,EAAE,IAAI,eAAe,EAAE;SACvC,CAAC;QACF,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACjC,WAAW,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,SAAS;YACf,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,OAAO,EAAE;gBACP,WAAW;gBACX,KAAK;gBACL,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;gBACpF,gBAAgB;aACjB;SACF,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,kBAAkB,CAAC;oBACvB,GAAG;oBACH,QAAQ;oBACR,2BAA2B,EAAE,IAAI,CAAC,kBAAkB;wBAClD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC;wBACjC,CAAC,CAAC,SAAS;oBACb,WAAW;oBACX,4BAA4B;oBAC5B,oBAAoB;oBACpB,qBAAqB;oBACrB,qBAAqB;oBACrB,KAAK;oBACL,MAAM,EAAE;wBACN,aAAa,EAAE,qBAAqB;wBACpC,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,kBAAkB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAChE;wBACD,iBAAiB,EAAE,IAAI,CAAC,GAAG,CACzB,IAAI,EACJ,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,CAAC,iBAAiB,IAAI,MAAM,CAAC,IAAI,MAAM,CAAC,CAC9E;qBACF;oBACD,gBAAgB;iBACjB,CAAC,CAAC;gBACH,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;oBACf,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,0BAA0B,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;wBAC5E,MAAM,SAAS,GAAG,6BAA6B,CAAC,QAAQ,CAAC,sBAAsB,EAAE;4BAC/E,aAAa,EAAE,CAAC;4BAChB,QAAQ;4BACR,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS;4BAC/B,WAAW,EAAE,GAAG,CAAC,EAAE;4BACnB,WAAW;4BACX,MAAM,EAAE,GAAG,CAAC,MAAM;yBACnB,CAAC,CAAC;wBACH,GAAG,CAAC,aAAa,GAAG,QAAQ,CAAC;wBAC7B,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;oBAClC,CAAC;oBAAC,OAAO,YAAqB,EAAE,CAAC;wBAC/B,WAAW,CAAC,GAAG,EAAE;4BACf,IAAI,EAAE,KAAK;4BACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BAC5B,OAAO,EAAE;gCACP,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,8DAA8D,YAAY,CACjF,YAAY,CACb,EAAE;6BACJ;yBACF,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBACD,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;gBACzB,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,WAAW;oBACjB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI;wBACpC,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;qBACzC;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;gBAC/E,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3C,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,OAAO;oBACb,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC;qBACzE;iBACF,CAAC,CAAC;YACL,CAAC;oBAAS,CAAC;gBACT,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO;oBAAE,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC/C,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACpB,UAAU,CAAC,GAAG,EAAE;oBACd,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClC,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,KAAK,4BAA4B,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAClE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;YACf,KAAK,EAAE,+BAA+B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC;SAC9E,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,6BAA6B,CAAC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC3E,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,oCAAoC,CACjD,2BAA2B,CAAC,QAAQ,CAAC,EACrC,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,6BAA6B,CAAC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC9E,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAG,sCAAsC,CACpD,2BAA2B,CAAC,QAAQ,CAAC,EACrC,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;QAC/C,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC5B,MAAM,KAAK,KAAK,EAChB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QACrB,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;QACnD,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;QAC3C,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAC1C,IAAI,cAAc,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACpE,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM;YAAE,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,GAAG,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;QAC/C,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC5B,MAAM,KAAK,KAAK,EAChB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,6BAA6B,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;QAClF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;QAC/C,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1B,MAAM,KAAK,MAAM,EACjB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,WAAW,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,KAAK;YACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,OAAO,EAAE;gBACP,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,gBAAgB;gBACxB,OAAO,EAAE,wBAAwB;aAClC;SACF,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,GAAG,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
1
|
+
{"version":3,"file":"tool-analysis.js","sourceRoot":"","sources":["../../src/app-server/tool-analysis.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EACL,+BAA+B,EAEhC,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,0BAA0B,EAE1B,sCAAsC,EAEtC,+BAA+B,EAE/B,oCAAoC,EACpC,6BAA6B,EAC9B,MAAM,4BAA4B,CAAC;AAapC,SAAS,gCAAgC,CAAC,aAAqB;IAC7D,OAAO,OAAO,CAAC,aAAa,EAAE,8BAA8B,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,mCAAmC,CAAC,aAAqB;IAChE,OAAO,OAAO,CAAC,aAAa,EAAE,8BAA8B,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,2BAA2B,CAAC,QAA4C;IAC/E,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,gCAAgC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,mCAAmC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC3E,IAAI,QAAQ,CAAC,sBAAsB,KAAK,WAAW,IAAI,MAAM,KAAK,WAAW;QAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjG,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,MAS9C;IACC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,IAAI,EAAE,GACzF,MAAM,CAAC;IACT,MAAM,EACJ,SAAS,EACT,MAAM,EACN,WAAW,EACX,YAAY,EACZ,aAAa,EACb,0BAA0B,EAC1B,kBAAkB,EACnB,GAAG,IAAI,CAAC;IAET,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,EAAE,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAElG,IAAI,QAAQ,KAAK,mCAAmC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC1E,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QACP,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,iDAAiD,EAAE,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CACrC,CAAC,UAAkB,EAAE,EAAE,CACrB,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,KAAK,0BAA0B,CAC3E,CAAC;QACF,IAAI,iBAAqE,CAAC;QAC1E,IAAI,CAAC;YACH,iBAAiB;gBACf,YAAY,CAAC,MAAM,GAAG,CAAC;oBACrB,CAAC,CAAC,MAAM,mBAAmB,CAAC,wBAAwB,CAAC,YAAY,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;oBACpF,CAAC,CAAC,SAAS,CAAC;QAClB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,+BAA+B,EAAE,CAAC;gBACrD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC/E,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,MAAM,0BAA0B,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,EAAE;YACxF,iBAAiB;SAClB,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClD,MAAM,wBAAwB,GAAG,GAAG,CAAC,wBAAwB,EAAE,CAAC;QAChE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;YACf,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC/B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,gBAAgB,EAAE,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI;gBAC7D,uBAAuB,EAAE,wBAAwB,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI;gBAC3E,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAChC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;oBACpB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;oBACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;oBAClC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;oBAClC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY;oBACpC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;iBAChD,CAAC,CAAC;aACJ,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,KAAK,yBAAyB,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,KAAK,GAAG;YACZ,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,cAAc,CAAC;YACpD,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,cAAc,CAAC;SACrD,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,iDAAiD,EAAE,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;YACnD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,mCAAmC,EAAE,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,qBAAqB,GACzB,IAAI,CAAC,qBAAqB,IAAI,OAAO,IAAI,CAAC,qBAAqB,KAAK,QAAQ;YAC1E,CAAC,CAAE,IAAI,CAAC,qBAAkD;YAC1D,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,IAAI,EAAE,CAAC;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CACvF,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,GAAG,GAAoB;YAC3B,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,IAAI,GAAG,EAAE;YAClB,eAAe,EAAE,IAAI,eAAe,EAAE;SACvC,CAAC;QACF,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACjC,WAAW,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,SAAS;YACf,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,OAAO,EAAE;gBACP,WAAW;gBACX,KAAK;gBACL,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;gBACpF,gBAAgB;aACjB;SACF,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,kBAAkB,CAAC;oBACvB,GAAG;oBACH,QAAQ;oBACR,2BAA2B,EAAE,IAAI,CAAC,kBAAkB;wBAClD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC;wBACjC,CAAC,CAAC,SAAS;oBACb,WAAW;oBACX,mBAAmB;oBACnB,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI;oBAC5B,qBAAqB;oBACrB,KAAK;oBACL,MAAM,EAAE;wBACN,aAAa,EAAE,qBAAqB;wBACpC,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,kBAAkB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAChE;wBACD,iBAAiB,EAAE,IAAI,CAAC,GAAG,CACzB,IAAI,EACJ,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,CAAC,iBAAiB,IAAI,MAAM,CAAC,IAAI,MAAM,CAAC,CAC9E;qBACF;oBACD,gBAAgB;iBACjB,CAAC,CAAC;gBACH,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;oBACf,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,0BAA0B,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;wBAC5E,MAAM,SAAS,GAAG,6BAA6B,CAAC,QAAQ,CAAC,sBAAsB,EAAE;4BAC/E,aAAa,EAAE,CAAC;4BAChB,QAAQ;4BACR,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS;4BAC/B,WAAW,EAAE,GAAG,CAAC,EAAE;4BACnB,WAAW;4BACX,MAAM,EAAE,GAAG,CAAC,MAAM;yBACnB,CAAC,CAAC;wBACH,GAAG,CAAC,aAAa,GAAG,QAAQ,CAAC;wBAC7B,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;oBAClC,CAAC;oBAAC,OAAO,YAAqB,EAAE,CAAC;wBAC/B,WAAW,CAAC,GAAG,EAAE;4BACf,IAAI,EAAE,KAAK;4BACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BAC5B,OAAO,EAAE;gCACP,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,8DAA8D,YAAY,CACjF,YAAY,CACb,EAAE;6BACJ;yBACF,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBACD,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;gBACzB,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,WAAW;oBACjB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI;wBACpC,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;qBACzC;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;gBAC/E,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3C,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,OAAO;oBACb,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC;qBACzE;iBACF,CAAC,CAAC;YACL,CAAC;oBAAS,CAAC;gBACT,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO;oBAAE,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC/C,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACpB,UAAU,CAAC,GAAG,EAAE;oBACd,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClC,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,KAAK,4BAA4B,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAClE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;YACf,KAAK,EAAE,+BAA+B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC;SAC9E,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,6BAA6B,CAAC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC3E,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,oCAAoC,CACjD,2BAA2B,CAAC,QAAQ,CAAC,EACrC,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,6BAA6B,CAAC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC9E,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAG,sCAAsC,CACpD,2BAA2B,CAAC,QAAQ,CAAC,EACrC,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;QAC/C,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC5B,MAAM,KAAK,KAAK,EAChB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QACrB,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;QACnD,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;QAC3C,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAC1C,IAAI,cAAc,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACpE,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM;YAAE,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,GAAG,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;QAC/C,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC5B,MAAM,KAAK,KAAK,EAChB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,6BAA6B,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;QAClF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;QAC/C,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1B,MAAM,KAAK,MAAM,EACjB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,WAAW,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,KAAK;YACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,OAAO,EAAE;gBACP,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,gBAAgB;gBACxB,OAAO,EAAE,wBAAwB;aAClC;SACF,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,GAAG,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { HealthMcpConnectionInfo } from '@inspectr/mcplab-core';
|
|
2
|
+
export type { HealthMcpConnectionInfo };
|
|
1
3
|
export interface AppServerOptions {
|
|
2
4
|
host: string;
|
|
3
5
|
port: number;
|
|
@@ -18,6 +20,11 @@ export interface AppSettings {
|
|
|
18
20
|
librariesDir: string;
|
|
19
21
|
scenarioAssistantAgentName?: string;
|
|
20
22
|
}
|
|
23
|
+
export interface HealthResponse {
|
|
24
|
+
ok: boolean;
|
|
25
|
+
version: string;
|
|
26
|
+
mcp: HealthMcpConnectionInfo;
|
|
27
|
+
}
|
|
21
28
|
export interface DevMcpServerRuntime {
|
|
22
29
|
host: string;
|
|
23
30
|
port: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/app-server/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,IAAI,CAAC;CAClB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/app-server/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAErE,YAAY,EAAE,uBAAuB,EAAE,CAAC;AAExC,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC;AAGD,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,uBAAuB,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,IAAI,CAAC;CAClB"}
|
package/dist/cli.js
CHANGED
|
@@ -4,7 +4,7 @@ dotenv.config();
|
|
|
4
4
|
import { Command } from 'commander';
|
|
5
5
|
import kleur from 'kleur';
|
|
6
6
|
import { readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
|
7
|
-
import { join, resolve } from 'node:path';
|
|
7
|
+
import { join, relative, resolve } from 'node:path';
|
|
8
8
|
import { loadConfig, hashConfig, selectScenarios, runAll, renderSummaryMarkdown, expandConfigForAgents } from '@inspectr/mcplab-core';
|
|
9
9
|
import { renderReport } from '@inspectr/mcplab-reporting';
|
|
10
10
|
import { execSync } from 'node:child_process';
|
|
@@ -14,6 +14,7 @@ import { readLibraries } from './app-server/libraries-store.js';
|
|
|
14
14
|
import { migrateSourceConfig } from './migrate-utils.js';
|
|
15
15
|
import { resolveRunOptions, runInteractiveSelection } from './run-interactive.js';
|
|
16
16
|
import { promptAppOptionsInteractive, selectRunDirInteractive } from './interactive-helpers.js';
|
|
17
|
+
import { deriveConfigRelativePath, resolveRunConfigSelection } from './eval-config-files.js';
|
|
17
18
|
import { applySnapshotPolicyToRunResult, buildSnapshotFromRun, compareRunToSnapshot, formatSnapshotComparisonTable, listSnapshots, loadSnapshot, saveSnapshot } from './snapshot.js';
|
|
18
19
|
const pkgVersion = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
|
|
19
20
|
?.version;
|
|
@@ -33,6 +34,7 @@ program
|
|
|
33
34
|
.option('--interactive', 'Prompt for required inputs')
|
|
34
35
|
.option('--snapshot-eval', 'Apply snapshot eval policy configured in the config')
|
|
35
36
|
.option('--compare-snapshot <snapshotId>', 'Compare completed run against snapshot id')
|
|
37
|
+
.option('--bail', 'Stop after first failed config when --config points to a folder')
|
|
36
38
|
.option('--run-note <text>', 'Optional note attached to the run metadata (max 500 chars)')
|
|
37
39
|
.option('--runs-dir <path>', 'Directory for run artifacts', 'mcplab/results/evaluation-runs')
|
|
38
40
|
.option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
|
|
@@ -66,118 +68,81 @@ program
|
|
|
66
68
|
agentsAll: Boolean(options.agentsAll),
|
|
67
69
|
interactiveSelection
|
|
68
70
|
});
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
71
|
+
const selection = resolveRunConfigSelection(resolvedOptions.config, process.cwd());
|
|
72
|
+
const configPaths = selection.configPaths;
|
|
73
|
+
const requestedPath = selection.requestedPath;
|
|
74
|
+
const requestedPathIsDirectory = selection.requestedPathIsDirectory;
|
|
75
|
+
const isBatch = requestedPathIsDirectory;
|
|
76
|
+
if (isBatch && options.compareSnapshot) {
|
|
77
|
+
throw new Error('--compare-snapshot is not supported when running a config folder');
|
|
76
78
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
? requestedAgentsFromCsv
|
|
87
|
-
: undefined;
|
|
88
|
-
const beforeExpandCount = config.scenarios.length;
|
|
89
|
-
const effectiveAgents = requestedAgents ?? config.run_defaults?.selected_agents;
|
|
90
|
-
const expanded = expandConfigForAgents(config, effectiveAgents);
|
|
91
|
-
if (expanded.scenarios.length !== beforeExpandCount || effectiveAgents?.length) {
|
|
92
|
-
const agentCount = effectiveAgents?.length ?? Object.keys(config.agents).length;
|
|
93
|
-
console.log(kleur.cyan(`๐ Testing ${beforeExpandCount} scenarios ร ${agentCount} selected agents = ${expanded.scenarios.length} total tests`));
|
|
94
|
-
}
|
|
95
|
-
const selected = selectScenarios(expanded, options.scenario);
|
|
96
|
-
const runsPerScenario = Number(options.runs);
|
|
97
|
-
if (Number.isNaN(runsPerScenario) || runsPerScenario <= 0) {
|
|
98
|
-
throw new Error('Runs must be a positive number');
|
|
99
|
-
}
|
|
100
|
-
const runNoteRaw = typeof options.runNote === 'string' ? String(options.runNote).trim() : '';
|
|
101
|
-
const runNote = runNoteRaw ? runNoteRaw.slice(0, 500) : undefined;
|
|
102
|
-
const oauthTokens = {};
|
|
103
|
-
for (const entry of options.oauthToken) {
|
|
104
|
-
const eqIdx = entry.indexOf('=');
|
|
105
|
-
if (eqIdx < 1) {
|
|
106
|
-
throw new Error(`Invalid --oauth-token format '${entry}'. Expected: server-name=token`);
|
|
107
|
-
}
|
|
108
|
-
const serverName = entry.slice(0, eqIdx).trim();
|
|
109
|
-
const token = entry.slice(eqIdx + 1).trim();
|
|
110
|
-
if (!serverName) {
|
|
111
|
-
throw new Error(`Invalid --oauth-token '${entry}': server name cannot be empty`);
|
|
79
|
+
if (!isBatch) {
|
|
80
|
+
const outcome = await executeSingleConfigRun({
|
|
81
|
+
configPath: configPaths[0],
|
|
82
|
+
options,
|
|
83
|
+
resolvedOptions
|
|
84
|
+
});
|
|
85
|
+
if (outcome.shouldFailOnDrift) {
|
|
86
|
+
console.error(kleur.red('Snapshot eval drift detected in fail_on_drift mode.'));
|
|
87
|
+
process.exit(2);
|
|
112
88
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
}
|
|
116
|
-
oauthTokens[serverName] = token;
|
|
89
|
+
console.log(kleur.gray('Process exiting.'));
|
|
90
|
+
return;
|
|
117
91
|
}
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
92
|
+
const rows = [];
|
|
93
|
+
let hadFailures = false;
|
|
94
|
+
for (let i = 0; i < configPaths.length; i += 1) {
|
|
95
|
+
const configPath = configPaths[i];
|
|
96
|
+
const displayPath = deriveConfigRelativePath(configPath, requestedPath);
|
|
97
|
+
console.log(kleur.cyan(`\nโถ Batch config ${i + 1}/${configPaths.length}: ${displayPath}`));
|
|
98
|
+
try {
|
|
99
|
+
const outcome = await executeSingleConfigRun({
|
|
100
|
+
configPath,
|
|
101
|
+
options,
|
|
102
|
+
resolvedOptions
|
|
103
|
+
});
|
|
104
|
+
const success = outcome.passed && !outcome.shouldFailOnDrift;
|
|
105
|
+
rows.push({
|
|
106
|
+
configPath: displayPath,
|
|
107
|
+
runId: outcome.runId,
|
|
108
|
+
runDir: outcome.runDir,
|
|
109
|
+
success
|
|
110
|
+
});
|
|
111
|
+
if (!success) {
|
|
112
|
+
hadFailures = true;
|
|
113
|
+
if (options.bail) {
|
|
114
|
+
console.error(kleur.red('Stopping batch due to --bail after failed config.'));
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
131
117
|
}
|
|
132
118
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
console.log(kleur.yellow('โ Snapshot eval enabled but no baseline snapshot is configured.'));
|
|
140
|
-
}
|
|
141
|
-
else {
|
|
142
|
-
const snapshot = loadSnapshot(String(policy.baseline_snapshot_id), resolve(options.snapshotsDir));
|
|
143
|
-
const comparison = compareRunToSnapshot(results, snapshot);
|
|
144
|
-
const enabledScenarioIds = new Set(selected.scenarios
|
|
145
|
-
.filter((scenario) => scenario.snapshot_eval?.enabled !== false)
|
|
146
|
-
.map((scenario) => scenario.id));
|
|
147
|
-
const applied = applySnapshotPolicyToRunResult({
|
|
148
|
-
results,
|
|
149
|
-
comparisons: [comparison],
|
|
150
|
-
policy,
|
|
151
|
-
enabledScenarioIds
|
|
119
|
+
catch (error) {
|
|
120
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
121
|
+
rows.push({
|
|
122
|
+
configPath: displayPath,
|
|
123
|
+
success: false,
|
|
124
|
+
error: message
|
|
152
125
|
});
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
console.log(kleur.yellow(`Impacted scenarios: ${applied.impacted_scenarios.join(', ')}`));
|
|
126
|
+
hadFailures = true;
|
|
127
|
+
if (options.bail) {
|
|
128
|
+
console.error(kleur.red('Stopping batch due to --bail after config error.'));
|
|
129
|
+
break;
|
|
158
130
|
}
|
|
159
|
-
console.log(formatSnapshotComparisonTable(comparison));
|
|
160
|
-
shouldFailOnDrift =
|
|
161
|
-
policy.mode === 'fail_on_drift' && applied.impacted_scenarios.length > 0;
|
|
162
131
|
}
|
|
163
132
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
const comparison = compareRunToSnapshot(results, snapshot);
|
|
174
|
-
console.log('');
|
|
175
|
-
console.log(kleur.cyan('๐ธ Snapshot Comparison'));
|
|
176
|
-
console.log(formatSnapshotComparisonTable(comparison));
|
|
133
|
+
console.log('');
|
|
134
|
+
console.log(kleur.cyan('Batch summary:'));
|
|
135
|
+
for (const row of rows) {
|
|
136
|
+
if (row.success) {
|
|
137
|
+
console.log(kleur.green(` โ ${row.configPath} (run=${row.runId ?? '-'}${row.runDir ? `, dir=${relative(process.cwd(), row.runDir)}` : ''})`));
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
console.log(kleur.red(` โ ${row.configPath}${row.error ? ` (${row.error})` : ''}`));
|
|
141
|
+
}
|
|
177
142
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
process.exit(
|
|
143
|
+
console.log(kleur.cyan(`Batch result: ${rows.filter((row) => row.success).length} succeeded, ${rows.filter((row) => !row.success).length} failed`));
|
|
144
|
+
if (hadFailures) {
|
|
145
|
+
process.exit(1);
|
|
181
146
|
}
|
|
182
147
|
console.log(kleur.gray('Process exiting.'));
|
|
183
148
|
}
|
|
@@ -607,6 +572,124 @@ program
|
|
|
607
572
|
}
|
|
608
573
|
});
|
|
609
574
|
program.parse();
|
|
575
|
+
async function executeSingleConfigRun(params) {
|
|
576
|
+
const { configPath, options, resolvedOptions } = params;
|
|
577
|
+
const loaded = loadConfig(resolve(configPath));
|
|
578
|
+
const { agents: libraryAgents } = readLibraries(loaded.bundleRoot);
|
|
579
|
+
loaded.config = { ...loaded.config, agents: { ...libraryAgents, ...loaded.config.agents } };
|
|
580
|
+
loaded.hash = hashConfig(loaded.config);
|
|
581
|
+
const { config, hash, warnings } = loaded;
|
|
582
|
+
for (const warning of warnings) {
|
|
583
|
+
console.log(kleur.yellow(`โ ${warning}`));
|
|
584
|
+
}
|
|
585
|
+
const requestedAgentsFromCsv = resolvedOptions.agents
|
|
586
|
+
? resolvedOptions.agents
|
|
587
|
+
.split(',')
|
|
588
|
+
.map((a) => a.trim())
|
|
589
|
+
.filter(Boolean)
|
|
590
|
+
: [];
|
|
591
|
+
const requestedAgents = resolvedOptions.agentsAll
|
|
592
|
+
? Object.keys(config.agents)
|
|
593
|
+
: requestedAgentsFromCsv.length > 0
|
|
594
|
+
? requestedAgentsFromCsv
|
|
595
|
+
: undefined;
|
|
596
|
+
const beforeExpandCount = config.scenarios.length;
|
|
597
|
+
const effectiveAgents = requestedAgents ?? config.run_defaults?.selected_agents;
|
|
598
|
+
const expanded = expandConfigForAgents(config, effectiveAgents);
|
|
599
|
+
if (expanded.scenarios.length !== beforeExpandCount || effectiveAgents?.length) {
|
|
600
|
+
const agentCount = effectiveAgents?.length ?? Object.keys(config.agents).length;
|
|
601
|
+
console.log(kleur.cyan(`๐ Testing ${beforeExpandCount} scenarios ร ${agentCount} selected agents = ${expanded.scenarios.length} total tests`));
|
|
602
|
+
}
|
|
603
|
+
const selected = selectScenarios(expanded, options.scenario);
|
|
604
|
+
const runsPerScenario = Number(options.runs);
|
|
605
|
+
if (Number.isNaN(runsPerScenario) || runsPerScenario <= 0) {
|
|
606
|
+
throw new Error('Runs must be a positive number');
|
|
607
|
+
}
|
|
608
|
+
const runNoteRaw = typeof options.runNote === 'string' ? String(options.runNote).trim() : '';
|
|
609
|
+
const runNote = runNoteRaw ? runNoteRaw.slice(0, 500) : undefined;
|
|
610
|
+
const oauthTokens = {};
|
|
611
|
+
for (const entry of options.oauthToken) {
|
|
612
|
+
const eqIdx = entry.indexOf('=');
|
|
613
|
+
if (eqIdx < 1) {
|
|
614
|
+
throw new Error(`Invalid --oauth-token format '${entry}'. Expected: server-name=token`);
|
|
615
|
+
}
|
|
616
|
+
const serverName = entry.slice(0, eqIdx).trim();
|
|
617
|
+
const token = entry.slice(eqIdx + 1).trim();
|
|
618
|
+
if (!serverName) {
|
|
619
|
+
throw new Error(`Invalid --oauth-token '${entry}': server name cannot be empty`);
|
|
620
|
+
}
|
|
621
|
+
if (!token) {
|
|
622
|
+
throw new Error(`Invalid --oauth-token '${entry}': token value cannot be empty`);
|
|
623
|
+
}
|
|
624
|
+
oauthTokens[serverName] = token;
|
|
625
|
+
}
|
|
626
|
+
const { runDir, results } = await runAll(selected, {
|
|
627
|
+
runsPerScenario,
|
|
628
|
+
scenarioId: options.scenario,
|
|
629
|
+
runNote,
|
|
630
|
+
configHash: hash,
|
|
631
|
+
gitCommit: getGitCommit(),
|
|
632
|
+
cliVersion: pkgVersion,
|
|
633
|
+
runsDir: String(options.runsDir),
|
|
634
|
+
oauthTokens: Object.keys(oauthTokens).length > 0 ? oauthTokens : undefined,
|
|
635
|
+
onProgress: async (event) => {
|
|
636
|
+
const line = formatRunProgressEvent(event);
|
|
637
|
+
if (line) {
|
|
638
|
+
console.log(`[${formatNowTime()}] ${line}`);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
});
|
|
642
|
+
let shouldFailOnDrift = false;
|
|
643
|
+
const useSnapshotEval = Boolean(options.snapshotEval) || Boolean(config.snapshot_eval?.enabled);
|
|
644
|
+
if (useSnapshotEval) {
|
|
645
|
+
const policy = config.snapshot_eval;
|
|
646
|
+
if (!policy?.baseline_snapshot_id) {
|
|
647
|
+
console.log(kleur.yellow('โ Snapshot eval enabled but no baseline snapshot is configured.'));
|
|
648
|
+
}
|
|
649
|
+
else {
|
|
650
|
+
const snapshot = loadSnapshot(String(policy.baseline_snapshot_id), resolve(options.snapshotsDir));
|
|
651
|
+
const comparison = compareRunToSnapshot(results, snapshot);
|
|
652
|
+
const enabledScenarioIds = new Set(selected.scenarios
|
|
653
|
+
.filter((scenario) => scenario.snapshot_eval?.enabled !== false)
|
|
654
|
+
.map((scenario) => scenario.id));
|
|
655
|
+
const applied = applySnapshotPolicyToRunResult({
|
|
656
|
+
results,
|
|
657
|
+
comparisons: [comparison],
|
|
658
|
+
policy,
|
|
659
|
+
enabledScenarioIds
|
|
660
|
+
});
|
|
661
|
+
console.log('');
|
|
662
|
+
console.log(kleur.cyan('๐ธ Snapshot Eval Policy'));
|
|
663
|
+
console.log(`${applied.mode} ยท baseline=${applied.baseline_snapshot_id} ยท overall=${applied.overall_score} ยท status=${applied.status}`);
|
|
664
|
+
if (applied.impacted_scenarios.length > 0) {
|
|
665
|
+
console.log(kleur.yellow(`Impacted scenarios: ${applied.impacted_scenarios.join(', ')}`));
|
|
666
|
+
}
|
|
667
|
+
console.log(formatSnapshotComparisonTable(comparison));
|
|
668
|
+
shouldFailOnDrift = policy.mode === 'fail_on_drift' && applied.impacted_scenarios.length > 0;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const reportPath = join(runDir, 'report.html');
|
|
672
|
+
const resultsPath = join(runDir, 'results.json');
|
|
673
|
+
const summaryPath = join(runDir, 'summary.md');
|
|
674
|
+
writeFileSync(resultsPath, `${JSON.stringify(results, null, 2)}\n`, 'utf8');
|
|
675
|
+
writeFileSync(reportPath, renderReport(results), 'utf8');
|
|
676
|
+
writeFileSync(summaryPath, renderSummaryMarkdown(results), 'utf8');
|
|
677
|
+
console.log(kleur.green(`โ
Run complete. Results: ${runDir}`));
|
|
678
|
+
if (options.compareSnapshot) {
|
|
679
|
+
const snapshot = loadSnapshot(String(options.compareSnapshot), resolve(options.snapshotsDir));
|
|
680
|
+
const comparison = compareRunToSnapshot(results, snapshot);
|
|
681
|
+
console.log('');
|
|
682
|
+
console.log(kleur.cyan('๐ธ Snapshot Comparison'));
|
|
683
|
+
console.log(formatSnapshotComparisonTable(comparison));
|
|
684
|
+
}
|
|
685
|
+
const failedRuns = results.scenarios.reduce((sum, scenario) => sum + scenario.runs.filter((run) => !run.pass).length, 0);
|
|
686
|
+
return {
|
|
687
|
+
runDir,
|
|
688
|
+
runId: results.metadata.run_id,
|
|
689
|
+
passed: failedRuns === 0,
|
|
690
|
+
shouldFailOnDrift
|
|
691
|
+
};
|
|
692
|
+
}
|
|
610
693
|
function getGitCommit() {
|
|
611
694
|
try {
|
|
612
695
|
const output = execSync('git rev-parse HEAD', { stdio: ['ignore', 'pipe', 'ignore'] })
|