@inspectr/mcplab 1.28.5 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -5
- package/dist/app/assets/index-256LN2cO.css +2 -0
- package/dist/app/assets/index-DVIaEnjQ.js +243 -0
- package/dist/app/assets/sse-queue-worker-Iwmqd2oW.js +1 -0
- package/dist/app/index.html +2 -2
- package/dist/app-server/app-context.d.ts +75 -115
- package/dist/app-server/app-context.d.ts.map +1 -1
- package/dist/app-server/app-dist.d.ts +3 -3
- package/dist/app-server/app-dist.js.map +1 -1
- package/dist/app-server/app-run-artifacts.d.ts +4 -0
- package/dist/app-server/app-run-artifacts.d.ts.map +1 -0
- package/dist/app-server/app-run-artifacts.js +66 -0
- package/dist/app-server/app-run-artifacts.js.map +1 -0
- package/dist/app-server/assistant-common.d.ts +27 -37
- package/dist/app-server/assistant-common.d.ts.map +1 -1
- package/dist/app-server/assistant-common.js +110 -1
- package/dist/app-server/assistant-common.js.map +1 -1
- package/dist/app-server/assistant-events.d.ts +8 -28
- package/dist/app-server/browser-launch.d.ts +3 -3
- package/dist/app-server/browser-launch.js.map +1 -1
- package/dist/app-server/browser-provider-learning.d.ts +26 -0
- package/dist/app-server/browser-provider-learning.d.ts.map +1 -0
- package/dist/app-server/browser-provider-learning.js +353 -0
- package/dist/app-server/browser-provider-learning.js.map +1 -0
- package/dist/app-server/config-store.d.ts +13 -21
- package/dist/app-server/dev-mcp.d.ts +2 -5
- package/dist/app-server/evals-routes.d.ts +8 -18
- package/dist/app-server/evaluation-group-results.d.ts +11 -0
- package/dist/app-server/evaluation-group-results.d.ts.map +1 -0
- package/dist/app-server/evaluation-group-results.js +46 -0
- package/dist/app-server/evaluation-group-results.js.map +1 -0
- package/dist/app-server/evaluation-journal-projection.d.ts +9 -0
- package/dist/app-server/evaluation-journal-projection.d.ts.map +1 -0
- package/dist/app-server/evaluation-journal-projection.js +76 -0
- package/dist/app-server/evaluation-journal-projection.js.map +1 -0
- package/dist/app-server/evaluation-journal-writer.d.ts +23 -0
- package/dist/app-server/evaluation-journal-writer.d.ts.map +1 -0
- package/dist/app-server/evaluation-journal-writer.js +41 -0
- package/dist/app-server/evaluation-journal-writer.js.map +1 -0
- package/dist/app-server/evaluation-result-projection.d.ts +12 -0
- package/dist/app-server/evaluation-result-projection.d.ts.map +1 -0
- package/dist/app-server/evaluation-result-projection.js +107 -0
- package/dist/app-server/evaluation-result-projection.js.map +1 -0
- package/dist/app-server/execution-journal.d.ts +13 -0
- package/dist/app-server/execution-journal.d.ts.map +1 -0
- package/dist/app-server/execution-journal.js +39 -0
- package/dist/app-server/execution-journal.js.map +1 -0
- package/dist/app-server/http.d.ts +1 -1
- package/dist/app-server/index.d.ts +1 -1
- package/dist/app-server/inspectr-session-manager.d.ts +45 -0
- package/dist/app-server/inspectr-session-manager.d.ts.map +1 -0
- package/dist/app-server/inspectr-session-manager.js +203 -0
- package/dist/app-server/inspectr-session-manager.js.map +1 -0
- package/dist/app-server/jobs.d.ts +6 -9
- package/dist/app-server/libraries-store.d.ts +26 -10
- package/dist/app-server/libraries-store.d.ts.map +1 -1
- package/dist/app-server/libraries-store.js +130 -3
- package/dist/app-server/libraries-store.js.map +1 -1
- package/dist/app-server/live-tests-routes.d.ts +17 -0
- package/dist/app-server/live-tests-routes.d.ts.map +1 -0
- package/dist/app-server/live-tests-routes.js +82 -0
- package/dist/app-server/live-tests-routes.js.map +1 -0
- package/dist/app-server/live-tests.d.ts +81 -0
- package/dist/app-server/live-tests.d.ts.map +1 -0
- package/dist/app-server/live-tests.js +272 -0
- package/dist/app-server/live-tests.js.map +1 -0
- package/dist/app-server/markdown-reports.d.ts +7 -7
- package/dist/app-server/oauth-debugger-domain.d.ts +183 -207
- package/dist/app-server/oauth-debugger-domain.js.map +1 -1
- package/dist/app-server/oauth-debugger.d.ts +10 -17
- package/dist/app-server/oauth-debugger.js +1 -1
- package/dist/app-server/oauth-debugger.js.map +1 -1
- package/dist/app-server/oauth-runtime-domain.d.ts +42 -53
- package/dist/app-server/oauth-runtime-domain.js.map +1 -1
- package/dist/app-server/oauth-runtime-routes.d.ts +12 -12
- package/dist/app-server/oauth-session-manager.d.ts +44 -50
- package/dist/app-server/oauth-session-manager.d.ts.map +1 -1
- package/dist/app-server/oauth-session-manager.js.map +1 -1
- package/dist/app-server/provider-models.d.ts +5 -5
- package/dist/app-server/result-assistant-domain.d.ts +66 -82
- package/dist/app-server/result-assistant-domain.d.ts.map +1 -1
- package/dist/app-server/result-assistant-domain.js.map +1 -1
- package/dist/app-server/result-assistant-tools.d.ts +5 -5
- package/dist/app-server/result-assistant.d.ts +9 -21
- package/dist/app-server/result-assistant.js.map +1 -1
- package/dist/app-server/router.d.ts +9 -1
- package/dist/app-server/router.d.ts.map +1 -1
- package/dist/app-server/router.js +328 -10
- package/dist/app-server/router.js.map +1 -1
- package/dist/app-server/rover-connection.d.ts +48 -0
- package/dist/app-server/rover-connection.d.ts.map +1 -0
- package/dist/app-server/rover-connection.js +172 -0
- package/dist/app-server/rover-connection.js.map +1 -0
- package/dist/app-server/run-agent-selection.d.ts +2 -5
- package/dist/app-server/run-queue-domain.d.ts +89 -65
- package/dist/app-server/run-queue-domain.d.ts.map +1 -1
- package/dist/app-server/run-queue-domain.js +723 -9
- package/dist/app-server/run-queue-domain.js.map +1 -1
- package/dist/app-server/run-queue-events.d.ts +3 -10
- package/dist/app-server/run-queue-events.d.ts.map +1 -1
- package/dist/app-server/run-queue-events.js +114 -14
- package/dist/app-server/run-queue-events.js.map +1 -1
- package/dist/app-server/run-queue-executor.d.ts +33 -47
- package/dist/app-server/run-queue-executor.d.ts.map +1 -1
- package/dist/app-server/run-queue-executor.js +85 -15
- package/dist/app-server/run-queue-executor.js.map +1 -1
- package/dist/app-server/run-queue-state.d.ts +81 -47
- package/dist/app-server/run-queue-state.d.ts.map +1 -1
- package/dist/app-server/run-queue-state.js.map +1 -1
- package/dist/app-server/runs-routes.d.ts +13 -35
- package/dist/app-server/runs-routes.d.ts.map +1 -1
- package/dist/app-server/runs-routes.js +182 -16
- package/dist/app-server/runs-routes.js.map +1 -1
- package/dist/app-server/runs-routes.test-helpers.d.ts +42 -38
- package/dist/app-server/runs-routes.test-helpers.d.ts.map +1 -1
- package/dist/app-server/runs-routes.test-helpers.js +4 -1
- package/dist/app-server/runs-routes.test-helpers.js.map +1 -1
- package/dist/app-server/runs-store.d.ts +60 -43
- package/dist/app-server/runs-store.d.ts.map +1 -1
- package/dist/app-server/runs-store.js +157 -70
- package/dist/app-server/runs-store.js.map +1 -1
- package/dist/app-server/scenario-assistant-domain.d.ts +131 -157
- package/dist/app-server/scenario-assistant-domain.d.ts.map +1 -1
- package/dist/app-server/scenario-assistant-domain.js +7 -15
- package/dist/app-server/scenario-assistant-domain.js.map +1 -1
- package/dist/app-server/scenario-assistant.d.ts +10 -26
- package/dist/app-server/scenario-assistant.js.map +1 -1
- package/dist/app-server/settings-store.d.ts +4 -4
- package/dist/app-server/static-serving.d.ts +7 -13
- package/dist/app-server/store-utils.d.ts +1 -1
- package/dist/app-server/tool-analysis-domain.d.ts +167 -183
- package/dist/app-server/tool-analysis-domain.d.ts.map +1 -1
- package/dist/app-server/tool-analysis-domain.js +2 -59
- package/dist/app-server/tool-analysis-domain.js.map +1 -1
- package/dist/app-server/tool-analysis-storage.d.ts +20 -34
- package/dist/app-server/tool-analysis.d.ts +10 -19
- package/dist/app-server/types.d.ts +25 -25
- package/dist/app-server/version-info.d.ts +6 -6
- package/dist/app-server/version-info.js.map +1 -1
- package/dist/app-server/workspace-paths.d.ts +2 -1
- package/dist/app-server/workspace-paths.d.ts.map +1 -1
- package/dist/app-server/workspace-paths.js +5 -1
- package/dist/app-server/workspace-paths.js.map +1 -1
- package/dist/cli-branding.d.ts +1 -1
- package/dist/cli-branding.js.map +1 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/eval-config-files.d.ts +5 -8
- package/dist/interactive-helpers.d.ts +11 -15
- package/dist/migrate-utils.d.ts +1 -1
- package/dist/results/context.d.ts +1 -1
- package/dist/results/format.d.ts +7 -14
- package/dist/results/indexer.d.ts +2 -9
- package/dist/results/search.d.ts +1 -1
- package/dist/results/types.d.ts +2 -11
- package/dist/run-interactive.d.ts +21 -23
- package/dist/test-results-fixture.d.ts +4 -4
- package/package.json +8 -6
- package/dist/app/assets/index-Bxq-wlTN.css +0 -1
- package/dist/app/assets/index-j_NpCqyK.js +0 -258
- package/dist/app/assets/sse-queue-worker-BMo6aPaB.js +0 -1
|
@@ -1,10 +1,115 @@
|
|
|
1
|
-
import { existsSync, readFileSync
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { tallyCheckCounts } from '@inspectr/mcplab-core';
|
|
3
|
+
import { listRunDirectories, snapshotRunDirectory, snapshotsEqual, tallyCheckCounts } from '@inspectr/mcplab-core';
|
|
4
4
|
import { ensureInsideRoot } from './store-utils.js';
|
|
5
|
+
export class RunStoreError extends Error {
|
|
6
|
+
statusCode;
|
|
7
|
+
constructor(statusCode, message) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.statusCode = statusCode;
|
|
10
|
+
this.name = 'RunStoreError';
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export class RunNotFoundError extends RunStoreError {
|
|
14
|
+
constructor(runId) {
|
|
15
|
+
super(404, `Run not found: ${runId}`);
|
|
16
|
+
this.name = 'RunNotFoundError';
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export class RunValidationError extends RunStoreError {
|
|
20
|
+
constructor(runId, message) {
|
|
21
|
+
super(422, `Invalid run ${runId}: ${message}`);
|
|
22
|
+
this.name = 'RunValidationError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const runSummaryCache = new Map();
|
|
26
|
+
function cacheFor(runsDir) {
|
|
27
|
+
const key = ensureInsideRoot(runsDir, runsDir);
|
|
28
|
+
const existing = runSummaryCache.get(key);
|
|
29
|
+
if (existing)
|
|
30
|
+
return existing;
|
|
31
|
+
const created = new Map();
|
|
32
|
+
runSummaryCache.set(key, created);
|
|
33
|
+
return created;
|
|
34
|
+
}
|
|
35
|
+
export function invalidateRunSummaryCache(runsDir, runId) {
|
|
36
|
+
cacheFor(runsDir).delete(runId);
|
|
37
|
+
}
|
|
38
|
+
function validateRunResults(runId, results) {
|
|
39
|
+
if (!results || typeof results !== 'object' || !results.metadata) {
|
|
40
|
+
throw new RunValidationError(runId, 'metadata is missing');
|
|
41
|
+
}
|
|
42
|
+
if (results.metadata.run_id !== runId) {
|
|
43
|
+
throw new RunValidationError(runId, `metadata.run_id must match the run directory name (received ${String(results.metadata.run_id)})`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function readAndValidateRunResults(runId, resultsPath) {
|
|
47
|
+
let parsed;
|
|
48
|
+
try {
|
|
49
|
+
parsed = JSON.parse(readFileSync(resultsPath, 'utf8'));
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
53
|
+
throw new RunValidationError(runId, `results.json could not be parsed (${message})`);
|
|
54
|
+
}
|
|
55
|
+
validateRunResults(runId, parsed);
|
|
56
|
+
return parsed;
|
|
57
|
+
}
|
|
58
|
+
function buildRunSummary(runId, dir, results) {
|
|
59
|
+
const scenarioItems = Array.isArray(results.scenarios) ? results.scenarios : [];
|
|
60
|
+
const checkCounts = tallyCheckCounts(scenarioItems.flatMap((scenario) => (scenario.runs ?? []).flatMap((run) => run.check_results ?? [])));
|
|
61
|
+
return {
|
|
62
|
+
runId,
|
|
63
|
+
evaluationRunId: results.metadata.evaluation_run_id ?? results.metadata.evaluation_group_id,
|
|
64
|
+
path: dir,
|
|
65
|
+
timestamp: results.metadata.timestamp,
|
|
66
|
+
runNote: results.metadata.run_note,
|
|
67
|
+
configHash: results.metadata.config_hash,
|
|
68
|
+
configPath: results.metadata.config_path,
|
|
69
|
+
configName: results.metadata.config_name,
|
|
70
|
+
mcpServerVersions: results.metadata.mcp_server_versions,
|
|
71
|
+
langsmithTraceUrls: results.metadata.langsmith_trace_urls,
|
|
72
|
+
toolTokensTotal: typeof results.metadata.tool_tokens_total === 'number'
|
|
73
|
+
? (results.metadata.tool_tokens_total ?? null)
|
|
74
|
+
: null,
|
|
75
|
+
scenarioIds: scenarioItems
|
|
76
|
+
.map((scenario) => String(scenario.scenario_id ?? ''))
|
|
77
|
+
.filter(Boolean),
|
|
78
|
+
scenarioNames: scenarioItems
|
|
79
|
+
.map((scenario) => String(scenario.scenario_name ?? ''))
|
|
80
|
+
.filter(Boolean),
|
|
81
|
+
agentIds: scenarioItems.map((scenario) => String(scenario.agent ?? '')).filter(Boolean),
|
|
82
|
+
rerunAgents: results.metadata.rerun_agents,
|
|
83
|
+
rerunScenarioIds: results.metadata.rerun_scenario_ids,
|
|
84
|
+
rerunServerOverrideAll: results.metadata.rerun_server_override_all,
|
|
85
|
+
rerunNewConversationBeforeStart: results.metadata.rerun_new_conversation_before_start,
|
|
86
|
+
rerunNewConversationBetweenScenarios: results.metadata.rerun_new_conversation_between_scenarios,
|
|
87
|
+
rerunScenarioServerOverrides: results.metadata.rerun_scenario_server_overrides,
|
|
88
|
+
totalScenarios: results.summary.total_scenarios,
|
|
89
|
+
totalRuns: results.summary.total_runs,
|
|
90
|
+
passRate: results.summary.total_runs <=
|
|
91
|
+
(results.summary.outcomes?.stopped ??
|
|
92
|
+
scenarioItems
|
|
93
|
+
.flatMap((scenario) => scenario.runs ?? [])
|
|
94
|
+
.filter((run) => run.outcome === 'stopped').length)
|
|
95
|
+
? null
|
|
96
|
+
: results.summary.pass_rate,
|
|
97
|
+
avgToolCalls: results.summary.avg_tool_calls_per_run,
|
|
98
|
+
avgLatencyMs: results.summary.avg_tool_latency_ms ?? 0,
|
|
99
|
+
totalToolDurationMs: typeof results.metadata.total_tool_duration_ms ===
|
|
100
|
+
'number'
|
|
101
|
+
? Math.max(0, results.metadata.total_tool_duration_ms ?? 0)
|
|
102
|
+
: undefined,
|
|
103
|
+
totalDurationMs: typeof results.metadata.total_duration_ms === 'number'
|
|
104
|
+
? Math.max(0, results.metadata.total_duration_ms ?? 0)
|
|
105
|
+
: undefined,
|
|
106
|
+
outcomes: results.summary.outcomes,
|
|
107
|
+
executionSource: results.metadata.execution_source,
|
|
108
|
+
executionClient: results.metadata.execution_client,
|
|
109
|
+
checkCounts
|
|
110
|
+
};
|
|
111
|
+
}
|
|
5
112
|
export function listRuns(runsDir, filter) {
|
|
6
|
-
if (!existsSync(runsDir))
|
|
7
|
-
return [];
|
|
8
113
|
const sinceParsedMs = filter?.since ? new Date(filter.since).getTime() : NaN;
|
|
9
114
|
const untilParsedMs = filter?.until ? new Date(filter.until).getTime() : NaN;
|
|
10
115
|
const sinceMsFromIso = Number.isFinite(sinceParsedMs) ? sinceParsedMs : Number.NEGATIVE_INFINITY;
|
|
@@ -15,83 +120,60 @@ export function listRuns(runsDir, filter) {
|
|
|
15
120
|
const sinceMsFromLastDays = lastDays !== null ? Date.now() - lastDays * 24 * 60 * 60 * 1000 : Number.NEGATIVE_INFINITY;
|
|
16
121
|
const sinceMs = Math.max(sinceMsFromIso, sinceMsFromLastDays);
|
|
17
122
|
const untilMs = untilMsFromIso;
|
|
18
|
-
const
|
|
123
|
+
const cache = cacheFor(runsDir);
|
|
124
|
+
const runIds = listRunDirectories(runsDir);
|
|
125
|
+
const currentRunIds = new Set(runIds);
|
|
126
|
+
for (const cachedRunId of cache.keys()) {
|
|
127
|
+
if (!currentRunIds.has(cachedRunId))
|
|
128
|
+
cache.delete(cachedRunId);
|
|
129
|
+
}
|
|
19
130
|
const summaries = [];
|
|
20
|
-
for (const
|
|
21
|
-
const
|
|
22
|
-
|
|
131
|
+
for (const runId of runIds) {
|
|
132
|
+
const dir = ensureInsideRoot(runsDir, join(runsDir, runId));
|
|
133
|
+
const snapshot = snapshotRunDirectory(runsDir, runId, ['results.json']);
|
|
134
|
+
if (!snapshot?.files['results.json']) {
|
|
135
|
+
cache.delete(runId);
|
|
23
136
|
continue;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
137
|
+
}
|
|
138
|
+
const cached = cache.get(runId);
|
|
139
|
+
let summary = cached?.summary;
|
|
140
|
+
if (!cached || !snapshotsEqual(cached.snapshot, snapshot)) {
|
|
141
|
+
try {
|
|
142
|
+
const results = readAndValidateRunResults(runId, ensureInsideRoot(runsDir, join(dir, 'results.json')));
|
|
143
|
+
summary = buildRunSummary(runId, dir, results);
|
|
144
|
+
cache.set(runId, { snapshot, summary });
|
|
29
145
|
}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
const matchesScenario = scenarioItems.some((scenario) => {
|
|
34
|
-
const scenarioId = String(scenario.scenario_id ?? '').trim();
|
|
35
|
-
const scenarioName = String(scenario.scenario_name ?? '').trim();
|
|
36
|
-
return scenarioId === needle || scenarioName === needle;
|
|
37
|
-
});
|
|
38
|
-
if (!matchesScenario)
|
|
39
|
-
continue;
|
|
146
|
+
catch {
|
|
147
|
+
cache.delete(runId);
|
|
148
|
+
continue;
|
|
40
149
|
}
|
|
41
|
-
const checkCounts = tallyCheckCounts(scenarioItems.flatMap((scenario) => (scenario.runs ?? []).flatMap((run) => run.check_results ?? [])));
|
|
42
|
-
summaries.push({
|
|
43
|
-
runId: results.metadata.run_id,
|
|
44
|
-
path: dir,
|
|
45
|
-
timestamp: results.metadata.timestamp,
|
|
46
|
-
runNote: results.metadata.run_note,
|
|
47
|
-
configHash: results.metadata.config_hash,
|
|
48
|
-
configPath: results.metadata.config_path,
|
|
49
|
-
configName: results.metadata.config_name,
|
|
50
|
-
mcpServerVersions: results.metadata.mcp_server_versions,
|
|
51
|
-
langsmithTraceUrls: results.metadata.langsmith_trace_urls,
|
|
52
|
-
toolTokensTotal: typeof results.metadata.tool_tokens_total ===
|
|
53
|
-
'number'
|
|
54
|
-
? results.metadata.tool_tokens_total ?? null
|
|
55
|
-
: null,
|
|
56
|
-
scenarioIds: scenarioItems
|
|
57
|
-
.map((scenario) => String(scenario.scenario_id ?? ''))
|
|
58
|
-
.filter(Boolean),
|
|
59
|
-
scenarioNames: scenarioItems
|
|
60
|
-
.map((scenario) => String(scenario.scenario_name ?? ''))
|
|
61
|
-
.filter(Boolean),
|
|
62
|
-
rerunAgents: results.metadata.rerun_agents,
|
|
63
|
-
rerunScenarioIds: results.metadata.rerun_scenario_ids,
|
|
64
|
-
rerunServerOverrideAll: results.metadata.rerun_server_override_all,
|
|
65
|
-
rerunScenarioServerOverrides: results.metadata.rerun_scenario_server_overrides,
|
|
66
|
-
totalScenarios: results.summary.total_scenarios,
|
|
67
|
-
totalRuns: results.summary.total_runs,
|
|
68
|
-
passRate: results.summary.pass_rate,
|
|
69
|
-
avgToolCalls: results.summary.avg_tool_calls_per_run,
|
|
70
|
-
avgLatencyMs: results.summary.avg_tool_latency_ms ?? 0,
|
|
71
|
-
totalToolDurationMs: typeof results.metadata
|
|
72
|
-
.total_tool_duration_ms === 'number'
|
|
73
|
-
? Math.max(0, results.metadata.total_tool_duration_ms ??
|
|
74
|
-
0)
|
|
75
|
-
: undefined,
|
|
76
|
-
totalDurationMs: typeof results.metadata.total_duration_ms ===
|
|
77
|
-
'number'
|
|
78
|
-
? Math.max(0, results.metadata.total_duration_ms ?? 0)
|
|
79
|
-
: undefined,
|
|
80
|
-
checkCounts
|
|
81
|
-
});
|
|
82
150
|
}
|
|
83
|
-
|
|
84
|
-
|
|
151
|
+
if (!summary)
|
|
152
|
+
continue;
|
|
153
|
+
const timestampMs = new Date(summary.timestamp).getTime();
|
|
154
|
+
if (!Number.isFinite(timestampMs) || timestampMs < sinceMs || timestampMs > untilMs) {
|
|
155
|
+
continue;
|
|
85
156
|
}
|
|
157
|
+
const scenarioItems = summary.scenarioIds ?? [];
|
|
158
|
+
if (filter?.scenario?.trim()) {
|
|
159
|
+
const needle = filter.scenario.trim();
|
|
160
|
+
const matchesScenario = scenarioItems.includes(needle) || (summary.scenarioNames ?? []).includes(needle);
|
|
161
|
+
if (!matchesScenario)
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
summaries.push(summary);
|
|
86
165
|
}
|
|
87
166
|
return summaries.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
|
|
88
167
|
}
|
|
89
168
|
export function getRunResults(runId, runsDir) {
|
|
169
|
+
if (!runId.trim() || runId.includes('/') || runId.includes('\\')) {
|
|
170
|
+
throw new RunStoreError(400, 'Invalid run id');
|
|
171
|
+
}
|
|
90
172
|
const runDir = ensureInsideRoot(runsDir, join(runsDir, runId));
|
|
91
173
|
const resultsPath = ensureInsideRoot(runsDir, join(runDir, 'results.json'));
|
|
92
|
-
if (!existsSync(resultsPath))
|
|
93
|
-
throw new
|
|
94
|
-
return
|
|
174
|
+
if (!existsSync(runDir) || !existsSync(resultsPath))
|
|
175
|
+
throw new RunNotFoundError(runId);
|
|
176
|
+
return readAndValidateRunResults(runId, resultsPath);
|
|
95
177
|
}
|
|
96
178
|
export function selectScenarioIds(config, requestedScenarioIds) {
|
|
97
179
|
if (!requestedScenarioIds || requestedScenarioIds.length === 0)
|
|
@@ -110,7 +192,12 @@ export function selectScenarioIds(config, requestedScenarioIds) {
|
|
|
110
192
|
}
|
|
111
193
|
return { ...config, scenarios };
|
|
112
194
|
}
|
|
113
|
-
export function getScenarioRunTraceRecords(runId, runsDir) {
|
|
195
|
+
export function getScenarioRunTraceRecords(runId, runsDir, options) {
|
|
196
|
+
if (options?.requireResults !== false)
|
|
197
|
+
getRunResults(runId, runsDir);
|
|
198
|
+
else if (!runId.trim() || runId.includes('/') || runId.includes('\\')) {
|
|
199
|
+
throw new RunStoreError(400, 'Invalid run id');
|
|
200
|
+
}
|
|
114
201
|
const runDir = ensureInsideRoot(runsDir, join(runsDir, runId));
|
|
115
202
|
const tracePath = ensureInsideRoot(runsDir, join(runDir, 'trace.jsonl'));
|
|
116
203
|
if (!existsSync(tracePath))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runs-store.js","sourceRoot":"","sources":["../../src/app-server/runs-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAyCpD,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,MAAuB;IAC/D,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,aAAa,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACjG,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACjG,MAAM,QAAQ,GACZ,OAAO,MAAM,EAAE,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;QAC7F,CAAC,CAAC,MAAM,CAAC,QAAQ;QACjB,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,mBAAmB,GACvB,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,cAAc,CAAC;IAC/B,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChD,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAC/C,CAAC;IACF,MAAM,SAAS,GAAiB,EAAE,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,SAAS;QACvC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAgB,CAAC;YAC7E,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;YACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,OAAO,IAAI,WAAW,GAAG,OAAO,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;YAChF,IAAI,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;oBACtD,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;oBACjE,OAAO,UAAU,KAAK,MAAM,IAAI,YAAY,KAAK,MAAM,CAAC;gBAC1D,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC,eAAe;oBAAE,SAAS;YACjC,CAAC;YACD,MAAM,WAAW,GAAG,gBAAgB,CAClC,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjC,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAChE,CACF,CAAC;YACF,SAAS,CAAC,IAAI,CAAC;gBACb,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAC9B,IAAI,EAAE,GAAG;gBACT,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;gBACrC,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAClC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;gBACxC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;gBACxC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;gBACxC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,mBAAmB;gBACvD,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,oBAAoB;gBACzD,eAAe,EACb,OAAQ,OAAO,CAAC,QAA4C,CAAC,iBAAiB;oBAC9E,QAAQ;oBACN,CAAC,CAAE,OAAO,CAAC,QAA2C,CAAC,iBAAiB,IAAI,IAAI;oBAChF,CAAC,CAAC,IAAI;gBACV,WAAW,EAAE,aAAa;qBACvB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;qBACrD,MAAM,CAAC,OAAO,CAAC;gBAClB,aAAa,EAAE,aAAa;qBACzB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;qBACvD,MAAM,CAAC,OAAO,CAAC;gBAClB,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAC1C,gBAAgB,EAAE,OAAO,CAAC,QAAQ,CAAC,kBAAkB;gBACrD,sBAAsB,EAAE,OAAO,CAAC,QAAQ,CAAC,yBAAyB;gBAClE,4BAA4B,EAAE,OAAO,CAAC,QAAQ,CAAC,+BAA+B;gBAC9E,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,eAAe;gBAC/C,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU;gBACrC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS;gBACnC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,sBAAsB;gBACpD,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC;gBACtD,mBAAmB,EACjB,OAAQ,OAAO,CAAC,QAAiD;qBAC9D,sBAAsB,KAAK,QAAQ;oBACpC,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,CAAC,EACA,OAAO,CAAC,QAAgD,CAAC,sBAAsB;wBAC9E,CAAC,CACJ;oBACH,CAAC,CAAC,SAAS;gBACf,eAAe,EACb,OAAQ,OAAO,CAAC,QAA4C,CAAC,iBAAiB;oBAC9E,QAAQ;oBACN,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,CAAC,EACA,OAAO,CAAC,QAA2C,CAAC,iBAAiB,IAAI,CAAC,CAC5E;oBACH,CAAC,CAAC,SAAS;gBACf,WAAW;aACZ,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,CACnB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAC5E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,OAAe;IAC1D,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IACzE,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAgB,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAkB,EAAE,oBAA+B;IACnF,IAAI,CAAC,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,sBAAsB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,SAAS;aACrE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAa,EACb,OAAe;IAEf,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1C,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YAC3D,IACE,MAAM;gBACN,MAAM,CAAC,IAAI,KAAK,cAAc;gBAC9B,MAAM,CAAC,aAAa,KAAK,CAAC;gBAC1B,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ;gBACtC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAChC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,MAA2C,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
1
|
+
{"version":3,"file":"runs-store.js","sourceRoot":"","sources":["../../src/app-server/runs-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,gBAAgB,EAGjB,MAAM,uBAAuB,CAAC;AAO/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,MAAM,OAAO,aAAc,SAAQ,KAAK;IAE3B,UAAU;IADrB,YACW,UAAkB,EAC3B,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;0BAHN,UAAU;QAInB,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED,MAAM,OAAO,gBAAiB,SAAQ,aAAa;IACjD,YAAY,KAAa;QACvB,KAAK,CAAC,GAAG,EAAE,kBAAkB,KAAK,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,aAAa;IACnD,YAAY,KAAa,EAAE,OAAe;QACxC,KAAK,CAAC,GAAG,EAAE,eAAe,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAuDD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAyC,CAAC;AAEzE,SAAS,QAAQ,CAAC,OAAe;IAC/B,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,OAAe,EAAE,KAAa;IACtE,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa,EAAE,OAAoB;IAC7D,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACjE,MAAM,IAAI,kBAAkB,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,kBAAkB,CAC1B,KAAK,EACL,+DAA+D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAClG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,KAAa,EAAE,WAAmB;IACnE,IAAI,MAAmB,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAgB,CAAC;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,IAAI,kBAAkB,CAAC,KAAK,EAAE,qCAAqC,OAAO,GAAG,CAAC,CAAC;IACvF,CAAC;IACD,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAoB;IACvE,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,MAAM,WAAW,GAAG,gBAAgB,CAClC,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjC,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAChE,CACF,CAAC;IACF,OAAO;QACL,KAAK;QACL,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,IAAI,OAAO,CAAC,QAAQ,CAAC,mBAAmB;QAC3F,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;QACrC,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ;QAClC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;QACxC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;QACxC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;QACxC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,mBAAmB;QACvD,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,oBAAoB;QACzD,eAAe,EACb,OAAQ,OAAO,CAAC,QAA4C,CAAC,iBAAiB,KAAK,QAAQ;YACzF,CAAC,CAAC,CAAE,OAAO,CAAC,QAA2C,CAAC,iBAAiB,IAAI,IAAI,CAAC;YAClF,CAAC,CAAC,IAAI;QACV,WAAW,EAAE,aAAa;aACvB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;aACrD,MAAM,CAAC,OAAO,CAAC;QAClB,aAAa,EAAE,aAAa;aACzB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;aACvD,MAAM,CAAC,OAAO,CAAC;QAClB,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;QACvF,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY;QAC1C,gBAAgB,EAAE,OAAO,CAAC,QAAQ,CAAC,kBAAkB;QACrD,sBAAsB,EAAE,OAAO,CAAC,QAAQ,CAAC,yBAAyB;QAClE,+BAA+B,EAAE,OAAO,CAAC,QAAQ,CAAC,mCAAmC;QACrF,oCAAoC,EAAE,OAAO,CAAC,QAAQ,CAAC,wCAAwC;QAC/F,4BAA4B,EAAE,OAAO,CAAC,QAAQ,CAAC,+BAA+B;QAC9E,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,eAAe;QAC/C,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU;QACrC,QAAQ,EACN,OAAO,CAAC,OAAO,CAAC,UAAU;YAC1B,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO;gBAChC,aAAa;qBACV,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;qBAC1C,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;YACrD,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS;QAC/B,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,sBAAsB;QACpD,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC;QACtD,mBAAmB,EACjB,OAAQ,OAAO,CAAC,QAAiD,CAAC,sBAAsB;YACxF,QAAQ;YACN,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,CAAC,EACA,OAAO,CAAC,QAAgD,CAAC,sBAAsB,IAAI,CAAC,CACtF;YACH,CAAC,CAAC,SAAS;QACf,eAAe,EACb,OAAQ,OAAO,CAAC,QAA4C,CAAC,iBAAiB,KAAK,QAAQ;YACzF,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAG,OAAO,CAAC,QAA2C,CAAC,iBAAiB,IAAI,CAAC,CAAC;YAC1F,CAAC,CAAC,SAAS;QACf,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,gBAAgB;QAClD,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,gBAAgB;QAClD,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,MAAuB;IAC/D,MAAM,aAAa,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,aAAa,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7E,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACjG,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACjG,MAAM,QAAQ,GACZ,OAAO,MAAM,EAAE,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;QAC7F,CAAC,CAAC,MAAM,CAAC,QAAQ;QACjB,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,mBAAmB,GACvB,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,cAAc,CAAC;IAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC;YAAE,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,SAAS,GAAiB,EAAE,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,oBAAoB,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YACrC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,OAAO,GAAG,MAAM,EAAE,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,yBAAyB,CACvC,KAAK,EACL,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CACrD,CAAC;gBACF,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC/C,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAC1C,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,OAAO,IAAI,WAAW,GAAG,OAAO,EAAE,CAAC;YACpF,SAAS;QACX,CAAC;QACD,MAAM,aAAa,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;QAChD,IAAI,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,eAAe,GACnB,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACnF,IAAI,CAAC,eAAe;gBAAE,SAAS;QACjC,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,CACnB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAC5E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,OAAe;IAC1D,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvF,OAAO,yBAAyB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAkB,EAAE,oBAA+B;IACnF,IAAI,CAAC,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC1C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,sBAAsB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,SAAS;aACrE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAa,EACb,OAAe,EACf,OAAsC;IAEtC,IAAI,OAAO,EAAE,cAAc,KAAK,KAAK;QAAE,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;SAChE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,aAAa,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1C,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YAC3D,IACE,MAAM;gBACN,MAAM,CAAC,IAAI,KAAK,cAAc;gBAC9B,MAAM,CAAC,aAAa,KAAK,CAAC;gBAC1B,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ;gBACtC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAChC,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,MAA2C,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1,184 +1,158 @@
|
|
|
1
1
|
import type { ServerResponse } from 'node:http';
|
|
2
|
-
import
|
|
2
|
+
import { type EvalConfig, type LlmAgentConfig, type LlmMessage, type ToolDef } from '@inspectr/mcplab-core';
|
|
3
3
|
import { McpClientManager } from '@inspectr/mcplab-core';
|
|
4
4
|
import type { AssistantSseEvent } from './assistant-events.js';
|
|
5
5
|
import { readLibraries } from './libraries-store.js';
|
|
6
6
|
export { truncateJson } from './assistant-common.js';
|
|
7
7
|
interface ScenarioAssistantContextInput {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
8
|
+
scenario: {
|
|
9
|
+
id: string;
|
|
10
|
+
name?: string;
|
|
11
|
+
prompt: string;
|
|
12
|
+
serverNames: string[];
|
|
13
|
+
evalRules: Array<{
|
|
14
|
+
type: string;
|
|
15
|
+
value?: string | number | boolean;
|
|
16
|
+
sequence?: string[];
|
|
17
|
+
path?: string;
|
|
18
|
+
equals?: string | number | boolean;
|
|
19
|
+
tool?: string;
|
|
20
|
+
label?: string;
|
|
21
|
+
prompt?: string;
|
|
22
|
+
}>;
|
|
23
|
+
extractRules: Array<{
|
|
24
|
+
name: string;
|
|
25
|
+
pattern: string;
|
|
26
|
+
}>;
|
|
27
|
+
};
|
|
28
|
+
availableServers?: Array<{
|
|
29
|
+
name: string;
|
|
30
|
+
url?: string;
|
|
22
31
|
}>;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
32
|
+
availableAgents?: Array<{
|
|
33
|
+
name: string;
|
|
34
|
+
provider: string;
|
|
35
|
+
model: string;
|
|
26
36
|
}>;
|
|
27
|
-
};
|
|
28
|
-
availableServers?: Array<{
|
|
29
|
-
name: string;
|
|
30
|
-
url?: string;
|
|
31
|
-
}>;
|
|
32
|
-
availableAgents?: Array<{
|
|
33
|
-
name: string;
|
|
34
|
-
provider: string;
|
|
35
|
-
model: string;
|
|
36
|
-
}>;
|
|
37
37
|
}
|
|
38
38
|
interface ScenarioAssistantSuggestionBundle {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
39
|
+
prompt?: {
|
|
40
|
+
replacement: string;
|
|
41
|
+
rationale?: string;
|
|
42
|
+
};
|
|
43
|
+
evalRules?: {
|
|
44
|
+
replacement: Array<{
|
|
45
|
+
type: string;
|
|
46
|
+
value?: string | number | boolean;
|
|
47
|
+
sequence?: string[];
|
|
48
|
+
path?: string;
|
|
49
|
+
equals?: string | number | boolean;
|
|
50
|
+
tool?: string;
|
|
51
|
+
label?: string;
|
|
52
|
+
prompt?: string;
|
|
53
|
+
}>;
|
|
54
|
+
rationale?: string;
|
|
55
|
+
};
|
|
56
|
+
extractRules?: {
|
|
57
|
+
replacement: Array<{
|
|
58
|
+
name: string;
|
|
59
|
+
pattern: string;
|
|
60
|
+
}>;
|
|
61
|
+
rationale?: string;
|
|
62
|
+
};
|
|
63
|
+
notes?: string[];
|
|
64
64
|
}
|
|
65
65
|
interface ScenarioAssistantEvalRuleSuggestion {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
66
|
+
type: string;
|
|
67
|
+
value?: string | number | boolean;
|
|
68
|
+
sequence?: string[];
|
|
69
|
+
path?: string;
|
|
70
|
+
equals?: string | number | boolean;
|
|
71
|
+
tool?: string;
|
|
72
|
+
label?: string;
|
|
73
|
+
prompt?: string;
|
|
74
74
|
}
|
|
75
75
|
interface AssistantPendingToolCall {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
76
|
+
id: string;
|
|
77
|
+
server: string;
|
|
78
|
+
tool: string;
|
|
79
|
+
publicToolName: string;
|
|
80
|
+
arguments: unknown;
|
|
81
|
+
status: 'pending' | 'approved' | 'denied' | 'error';
|
|
82
|
+
createdAt: string;
|
|
83
|
+
resultPreview?: string;
|
|
84
|
+
error?: string;
|
|
85
85
|
}
|
|
86
86
|
interface AssistantChatMessage {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
87
|
+
id: string;
|
|
88
|
+
role: 'user' | 'assistant' | 'tool' | 'system';
|
|
89
|
+
text: string;
|
|
90
|
+
createdAt: string;
|
|
91
|
+
suggestions?: ScenarioAssistantSuggestionBundle;
|
|
92
|
+
pendingToolCallId?: string;
|
|
93
|
+
pendingToolCallIds?: string[];
|
|
94
|
+
toolRequestServer?: string;
|
|
95
|
+
toolRequestName?: string;
|
|
96
|
+
toolRequestPublicName?: string;
|
|
97
97
|
}
|
|
98
98
|
export interface ScenarioAssistantSession {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
systemPromptCache?: string;
|
|
120
|
-
events: AssistantSseEvent[];
|
|
121
|
-
clients: Set<ServerResponse>;
|
|
99
|
+
id: string;
|
|
100
|
+
createdAt: number;
|
|
101
|
+
lastTouchedAt: number;
|
|
102
|
+
configPath?: string;
|
|
103
|
+
selectedAssistantAgentName: string;
|
|
104
|
+
context: ScenarioAssistantContextInput;
|
|
105
|
+
agentConfig: LlmAgentConfig;
|
|
106
|
+
mcp: McpClientManager;
|
|
107
|
+
tools: ToolDef[];
|
|
108
|
+
toolPublicMap: Map<string, {
|
|
109
|
+
server: string;
|
|
110
|
+
tool: string;
|
|
111
|
+
}>;
|
|
112
|
+
pendingToolCalls: AssistantPendingToolCall[];
|
|
113
|
+
chatMessages: AssistantChatMessage[];
|
|
114
|
+
llmMessages: LlmMessage[];
|
|
115
|
+
warnings: string[];
|
|
116
|
+
systemPromptCache?: string;
|
|
117
|
+
events: AssistantSseEvent[];
|
|
118
|
+
clients: Set<ServerResponse>;
|
|
122
119
|
}
|
|
123
|
-
export declare function cleanupAssistantSessions(
|
|
124
|
-
sessions: Map<string, ScenarioAssistantSession>,
|
|
125
|
-
now?: number
|
|
126
|
-
): void;
|
|
120
|
+
export declare function cleanupAssistantSessions(sessions: Map<string, ScenarioAssistantSession>, now?: number): void;
|
|
127
121
|
export declare function touchAssistantSession(session: ScenarioAssistantSession): void;
|
|
128
122
|
export declare function assistantSessionView(session: ScenarioAssistantSession): {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
123
|
+
id: string;
|
|
124
|
+
createdAt: string;
|
|
125
|
+
updatedAt: string;
|
|
126
|
+
selectedAssistantAgentName: string;
|
|
127
|
+
model: string;
|
|
128
|
+
provider: "anthropic" | "azure_openai" | "openai";
|
|
129
|
+
warnings: string[];
|
|
130
|
+
toolsLoaded: number;
|
|
131
|
+
toolServers: string[];
|
|
132
|
+
messages: AssistantChatMessage[];
|
|
133
|
+
pendingToolCalls: AssistantPendingToolCall[];
|
|
140
134
|
};
|
|
141
|
-
export declare function preloadAssistantTools(
|
|
142
|
-
session: ScenarioAssistantSession,
|
|
143
|
-
serversByName: Record<string, EvalConfig['servers'][string]>,
|
|
144
|
-
selectedServerNames: string[],
|
|
145
|
-
options?: {
|
|
135
|
+
export declare function preloadAssistantTools(session: ScenarioAssistantSession, serversByName: Record<string, EvalConfig['servers'][string]>, selectedServerNames: string[], options?: {
|
|
146
136
|
serverAuthHeaders?: Record<string, Record<string, string>>;
|
|
147
|
-
|
|
148
|
-
):
|
|
149
|
-
export declare function
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
type: 'assistant_message' | 'tool_call_request';
|
|
159
|
-
text: string;
|
|
160
|
-
suggestions?: ScenarioAssistantSuggestionBundle;
|
|
161
|
-
pendingToolCall?: AssistantPendingToolCall;
|
|
162
|
-
pendingToolCalls?: AssistantPendingToolCall[];
|
|
163
|
-
};
|
|
137
|
+
}): Promise<void>;
|
|
138
|
+
export declare function normalizeScenarioAssistantEvalRules(replacement: ScenarioAssistantEvalRuleSuggestion[]): ScenarioAssistantEvalRuleSuggestion[];
|
|
139
|
+
export declare function continueAssistantTurn(session: ScenarioAssistantSession, signal?: AbortSignal): Promise<{
|
|
140
|
+
session: ReturnType<typeof assistantSessionView>;
|
|
141
|
+
response: {
|
|
142
|
+
type: 'assistant_message' | 'tool_call_request';
|
|
143
|
+
text: string;
|
|
144
|
+
suggestions?: ScenarioAssistantSuggestionBundle;
|
|
145
|
+
pendingToolCall?: AssistantPendingToolCall;
|
|
146
|
+
pendingToolCalls?: AssistantPendingToolCall[];
|
|
147
|
+
};
|
|
164
148
|
}>;
|
|
165
|
-
export declare function executeAssistantToolCall(
|
|
166
|
-
session: ScenarioAssistantSession,
|
|
167
|
-
pending: AssistantPendingToolCall,
|
|
168
|
-
signal?: AbortSignal
|
|
169
|
-
): Promise<unknown>;
|
|
149
|
+
export declare function executeAssistantToolCall(session: ScenarioAssistantSession, pending: AssistantPendingToolCall, signal?: AbortSignal): Promise<unknown>;
|
|
170
150
|
export declare function summarizeToolResultForAssistant(result: unknown): string;
|
|
171
|
-
export declare function resolveAssistantAgentFromConfig(
|
|
172
|
-
|
|
173
|
-
selectedAssistantAgentName: string
|
|
174
|
-
): AgentConfig;
|
|
175
|
-
export declare function resolveAssistantAgentFromLibraries(
|
|
176
|
-
libraries: ReturnType<typeof readLibraries>,
|
|
177
|
-
selectedAssistantAgentName: string
|
|
178
|
-
): AgentConfig;
|
|
151
|
+
export declare function resolveAssistantAgentFromConfig(config: EvalConfig, selectedAssistantAgentName: string): LlmAgentConfig;
|
|
152
|
+
export declare function resolveAssistantAgentFromLibraries(libraries: ReturnType<typeof readLibraries>, selectedAssistantAgentName: string): LlmAgentConfig;
|
|
179
153
|
export declare function pickDefaultAssistantAgentName(params: {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
154
|
+
requested?: string;
|
|
155
|
+
settingsDefault?: string;
|
|
156
|
+
agentNames: string[];
|
|
183
157
|
}): string;
|
|
184
|
-
//# sourceMappingURL=scenario-assistant-domain.d.ts.map
|
|
158
|
+
//# sourceMappingURL=scenario-assistant-domain.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario-assistant-domain.d.ts","sourceRoot":"","sources":["../../src/app-server/scenario-assistant-domain.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"scenario-assistant-domain.d.ts","sourceRoot":"","sources":["../../src/app-server/scenario-assistant-domain.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,OAAO,EACb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAYzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,UAAU,6BAA6B;IACrC,QAAQ,EAAE;QACR,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,SAAS,EAAE,KAAK,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YAClC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YACnC,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;SACjB,CAAC,CAAC;QACH,YAAY,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACxD,CAAC;IACF,gBAAgB,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzD,eAAe,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC5E;AAED,UAAU,iCAAiC;IACzC,MAAM,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,KAAK,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YAClC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YACnC,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;SACjB,CAAC,CAAC;QACH,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,YAAY,CAAC,EAAE;QACb,WAAW,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtD,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAcD,UAAU,wBAAwB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,oBAAoB;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,iCAAiC,CAAC;IAChD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0BAA0B,EAAE,MAAM,CAAC;IACnC,OAAO,EAAE,6BAA6B,CAAC;IACvC,WAAW,EAAE,cAAc,CAAC;IAC5B,GAAG,EAAE,gBAAgB,CAAC;IACtB,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC7D,gBAAgB,EAAE,wBAAwB,EAAE,CAAC;IAC7C,YAAY,EAAE,oBAAoB,EAAE,CAAC;IACrC,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,MAAM,EAAE,iBAAiB,EAAE,CAAC;IAC5B,OAAO,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC;CAC9B;AAMD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,wBAAwB,CAAC,EAC/C,GAAG,SAAa,GACf,IAAI,CAEN;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,wBAAwB,GAAG,IAAI,CAE7E;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,wBAAwB;IAElE,EAAE;IACF,SAAS;IACT,SAAS;IACT,0BAA0B;IAC1B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,WAAW;IACX,QAAQ;IACR,gBAAgB;EAEnB;AA2ED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,wBAAwB,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,EAC5D,mBAAmB,EAAE,MAAM,EAAE,EAC7B,OAAO,CAAC,EAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAAE,GACvE,OAAO,CAAC,IAAI,CAAC,CA0Bf;AA0SD,wBAAgB,mCAAmC,CACjD,WAAW,EAAE,mCAAmC,EAAE,GACjD,mCAAmC,EAAE,CAwCvC;AA4DD,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC;IACT,OAAO,EAAE,UAAU,CAAC,OAAO,oBAAoB,CAAC,CAAC;IACjD,QAAQ,EAAE;QACR,IAAI,EAAE,mBAAmB,GAAG,mBAAmB,CAAC;QAChD,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,iCAAiC,CAAC;QAChD,eAAe,CAAC,EAAE,wBAAwB,CAAC;QAC3C,gBAAgB,CAAC,EAAE,wBAAwB,EAAE,CAAC;KAC/C,CAAC;CACH,CAAC,CAiGD;AAED,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,wBAAwB,EACjC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,OAAO,CAAC,CAWlB;AAED,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAEvE;AAED,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,UAAU,EAClB,0BAA0B,EAAE,MAAM,GACjC,cAAc,CAYhB;AAED,wBAAgB,kCAAkC,CAChD,SAAS,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,EAC3C,0BAA0B,EAAE,MAAM,GACjC,cAAc,CAYhB;AAED,wBAAgB,6BAA6B,CAAC,MAAM,EAAE;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB,GAAG,MAAM,CAMT"}
|