@evalops/maestro 0.10.11 → 0.10.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/transport/tool-execution-bridge.d.ts.map +1 -1
- package/dist/agent/transport/tool-execution-bridge.js +4 -0
- package/dist/agent/transport/tool-execution-bridge.js.map +1 -1
- package/dist/audit/logger.d.ts +1 -1
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +5 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/commands/config.d.ts.map +1 -1
- package/dist/cli/commands/config.js +3 -2
- package/dist/cli/commands/config.js.map +1 -1
- package/dist/cli/commands/evalops.d.ts +2 -1
- package/dist/cli/commands/evalops.d.ts.map +1 -1
- package/dist/cli/commands/evalops.js +9 -29
- package/dist/cli/commands/evalops.js.map +1 -1
- package/dist/cli/commands/init.d.ts +3 -1
- package/dist/cli/commands/init.d.ts.map +1 -1
- package/dist/cli/commands/init.js +61 -21
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/status.d.ts +2 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +11 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +3 -0
- package/dist/cli/help.js.map +1 -1
- package/dist/cli-tui/tui-renderer/attachment-controller.d.ts.map +1 -1
- package/dist/cli-tui/tui-renderer/attachment-controller.js +18 -1
- package/dist/cli-tui/tui-renderer/attachment-controller.js.map +1 -1
- package/dist/cli.js +2316 -1716
- package/dist/evalops/agent-bootstrap.d.ts +9 -0
- package/dist/evalops/agent-bootstrap.d.ts.map +1 -1
- package/dist/evalops/agent-bootstrap.js +254 -36
- package/dist/evalops/agent-bootstrap.js.map +1 -1
- package/dist/evalops/managed-context.d.ts +34 -0
- package/dist/evalops/managed-context.d.ts.map +1 -0
- package/dist/evalops/managed-context.js +183 -0
- package/dist/evalops/managed-context.js.map +1 -0
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +6 -1
- package/dist/main.js.map +1 -1
- package/dist/memory/auto-extraction.d.ts.map +1 -1
- package/dist/memory/auto-extraction.js +58 -0
- package/dist/memory/auto-extraction.js.map +1 -1
- package/dist/memory/backend.d.ts +1 -0
- package/dist/memory/backend.d.ts.map +1 -1
- package/dist/memory/backend.js.map +1 -1
- package/dist/memory/service-client.d.ts +1 -0
- package/dist/memory/service-client.d.ts.map +1 -1
- package/dist/memory/service-client.js +32 -8
- package/dist/memory/service-client.js.map +1 -1
- package/dist/models/builtin.d.ts.map +1 -1
- package/dist/models/builtin.js +2 -3
- package/dist/models/builtin.js.map +1 -1
- package/dist/oauth/evalops.d.ts.map +1 -1
- package/dist/oauth/evalops.js +2 -1
- package/dist/oauth/evalops.js.map +1 -1
- package/dist/oauth/storage.d.ts +1 -0
- package/dist/oauth/storage.d.ts.map +1 -1
- package/dist/oauth/storage.js +6 -0
- package/dist/oauth/storage.js.map +1 -1
- package/dist/platform/client.d.ts.map +1 -1
- package/dist/platform/client.js +3 -0
- package/dist/platform/client.js.map +1 -1
- package/dist/platform/core-services.d.ts +1 -0
- package/dist/platform/core-services.d.ts.map +1 -1
- package/dist/platform/core-services.js +1 -0
- package/dist/platform/core-services.js.map +1 -1
- package/dist/platform/tool-execution-client.d.ts.map +1 -1
- package/dist/platform/tool-execution-client.js +3 -0
- package/dist/platform/tool-execution-client.js.map +1 -1
- package/dist/providers/auth.d.ts +1 -1
- package/dist/providers/auth.d.ts.map +1 -1
- package/dist/providers/auth.js +30 -2
- package/dist/providers/auth.js.map +1 -1
- package/dist/providers/evalops-managed.d.ts +1 -0
- package/dist/providers/evalops-managed.d.ts.map +1 -1
- package/dist/providers/evalops-managed.js +1 -0
- package/dist/providers/evalops-managed.js.map +1 -1
- package/dist/telemetry/maestro-event-bus.d.ts.map +1 -1
- package/dist/telemetry/maestro-event-bus.js +16 -47
- package/dist/telemetry/maestro-event-bus.js.map +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +98 -2
- package/dist/telemetry.js.map +1 -1
- package/dist/version.json +2 -2
- package/node_modules/@evalops/contracts/package.json +1 -1
- package/node_modules/@evalops/tui/package.json +1 -1
- package/package.json +4 -4
|
@@ -38,15 +38,24 @@ export interface EvalOpsAgentMcpMetadata {
|
|
|
38
38
|
export interface EvalOpsInitResult {
|
|
39
39
|
agentId?: string;
|
|
40
40
|
apiKeyCreated: boolean;
|
|
41
|
+
approvalPolicyAttached?: boolean;
|
|
42
|
+
authenticatedAs?: string;
|
|
43
|
+
consoleUrl?: string;
|
|
41
44
|
endpoint: string;
|
|
45
|
+
evidenceEventPublished?: boolean;
|
|
46
|
+
evidenceEvents?: number;
|
|
47
|
+
governedActionsLoaded?: number;
|
|
48
|
+
governedInferenceCheckRan?: boolean;
|
|
42
49
|
keyPrefix?: string;
|
|
43
50
|
manifestUrl?: string;
|
|
44
51
|
organizationId?: string;
|
|
45
52
|
registryVisible?: boolean;
|
|
53
|
+
riskFindings?: number;
|
|
46
54
|
runId?: string;
|
|
47
55
|
scopesGranted?: string[];
|
|
48
56
|
sessionExpiresAt?: string;
|
|
49
57
|
stored: boolean;
|
|
58
|
+
traceIngestionStarted?: boolean;
|
|
50
59
|
}
|
|
51
60
|
export interface EvalOpsInitStatus {
|
|
52
61
|
message: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-bootstrap.d.ts","sourceRoot":"","sources":["../../src/evalops/agent-bootstrap.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAEN,oBAAoB,EACpB,oBAAoB,EACpB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"agent-bootstrap.d.ts","sourceRoot":"","sources":["../../src/evalops/agent-bootstrap.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAEN,oBAAoB,EACpB,oBAAoB,EACpB,MAAM,qBAAqB,CAAC;AAqB7B,MAAM,WAAW,kBAAkB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IACjC,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACvC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,gBAAgB,CAAC;IAC9E,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,aAAa,CAAC;IACrC,mBAAmB,CAAC,EAAE,OAAO,mBAAmB,CAAC;IACjD,eAAe,CAAC,EAAE,OAAO,oBAAoB,CAAC;IAC9C,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC/C,eAAe,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC9C;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CACP,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAmHD,wBAAgB,gCAAgC,IAC7C,uBAAuB,GACvB,SAAS,CAEX;AAmgBD,wBAAsB,qBAAqB,CAC1C,OAAO,GAAE,kBAAuB,EAChC,YAAY,GAAE,uBAA4B,GACxC,OAAO,CAAC,iBAAiB,CAAC,CA+J5B"}
|
|
@@ -5,17 +5,20 @@ import { getOAuthToken, hasOAuthCredentials, login } from "../oauth/index.js";
|
|
|
5
5
|
import { loadOAuthCredentials, saveOAuthCredentials, } from "../oauth/storage.js";
|
|
6
6
|
import { getPackageVersion } from "../package-metadata.js";
|
|
7
7
|
import { getEnvValue, normalizeBaseUrl } from "../platform/client.js";
|
|
8
|
+
import { PLATFORM_HTTP_ROUTES } from "../platform/core-services.js";
|
|
8
9
|
const DEFAULT_AGENT_MCP_BASE_URL = "https://app.evalops.dev";
|
|
10
|
+
const DEFAULT_IDENTITY_BASE_URL = "https://identity.evalops.dev";
|
|
9
11
|
const AGENT_MCP_MANIFEST_PATH = "/.well-known/evalops/agent-mcp.json";
|
|
10
12
|
const AGENT_MCP_PATH = "/mcp";
|
|
11
13
|
const DEFAULT_AGENT_TYPE = "maestro";
|
|
12
14
|
const DEFAULT_SURFACE = "cli";
|
|
13
|
-
const DEFAULT_REGISTER_SCOPES = ["llm_gateway:invoke"];
|
|
14
15
|
const DEFAULT_API_KEY_SCOPES = [
|
|
15
16
|
"agent:register",
|
|
16
17
|
"agent:heartbeat",
|
|
17
18
|
"governance:evaluate",
|
|
19
|
+
"llm_gateway:invoke",
|
|
18
20
|
"memories:read",
|
|
21
|
+
"memories:write",
|
|
19
22
|
"meter:record",
|
|
20
23
|
];
|
|
21
24
|
function isRecord(value) {
|
|
@@ -96,6 +99,53 @@ function normalizeManifestUrl(url) {
|
|
|
96
99
|
}
|
|
97
100
|
return parsed.toString();
|
|
98
101
|
}
|
|
102
|
+
function resolveIdentityBaseUrl(credentials, endpoint) {
|
|
103
|
+
const configured = getEnvValue([
|
|
104
|
+
"MAESTRO_IDENTITY_URL",
|
|
105
|
+
"EVALOPS_IDENTITY_URL",
|
|
106
|
+
"MAESTRO_PLATFORM_BASE_URL",
|
|
107
|
+
"MAESTRO_EVALOPS_BASE_URL",
|
|
108
|
+
"EVALOPS_BASE_URL",
|
|
109
|
+
]);
|
|
110
|
+
const stored = typeof credentials?.metadata?.identityBaseUrl === "string"
|
|
111
|
+
? credentials.metadata.identityBaseUrl
|
|
112
|
+
: undefined;
|
|
113
|
+
const derived = identityBaseUrlFromMcpEndpoint(endpoint?.endpoint);
|
|
114
|
+
const storedBeforeDerived = endpoint
|
|
115
|
+
? !endpoint.preferDerivedIdentity && isCustomMcpEndpoint(endpoint.endpoint)
|
|
116
|
+
: false;
|
|
117
|
+
return normalizeBaseUrl(configured ??
|
|
118
|
+
endpoint?.identityBaseUrl ??
|
|
119
|
+
(endpoint?.preferDerivedIdentity ? derived : undefined) ??
|
|
120
|
+
(storedBeforeDerived ? stored : undefined) ??
|
|
121
|
+
derived ??
|
|
122
|
+
stored ??
|
|
123
|
+
DEFAULT_IDENTITY_BASE_URL, Object.values(PLATFORM_HTTP_ROUTES.identity));
|
|
124
|
+
}
|
|
125
|
+
function isCustomMcpEndpoint(endpoint) {
|
|
126
|
+
const parsed = new URL(normalizeBaseUrl(endpoint));
|
|
127
|
+
return (parsed.hostname !== "app.evalops.dev" &&
|
|
128
|
+
parsed.hostname !== "staging.evalops.dev");
|
|
129
|
+
}
|
|
130
|
+
function identityBaseUrlFromMcpEndpoint(endpoint) {
|
|
131
|
+
if (!endpoint) {
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
const parsed = new URL(normalizeBaseUrl(endpoint));
|
|
135
|
+
if (parsed.hostname === "app.evalops.dev") {
|
|
136
|
+
return DEFAULT_IDENTITY_BASE_URL;
|
|
137
|
+
}
|
|
138
|
+
if (parsed.hostname === "staging.evalops.dev") {
|
|
139
|
+
return "https://api.staging.evalops.dev";
|
|
140
|
+
}
|
|
141
|
+
if (parsed.hostname.startsWith("app.")) {
|
|
142
|
+
parsed.hostname = `identity.${parsed.hostname.slice("app.".length)}`;
|
|
143
|
+
}
|
|
144
|
+
parsed.pathname = "";
|
|
145
|
+
parsed.search = "";
|
|
146
|
+
parsed.hash = "";
|
|
147
|
+
return normalizeBaseUrl(parsed.toString());
|
|
148
|
+
}
|
|
99
149
|
async function resolveEndpointFromManifest(manifestUrl, fetchImpl) {
|
|
100
150
|
const response = await fetchImpl(manifestUrl, {
|
|
101
151
|
headers: { accept: "application/json" },
|
|
@@ -104,18 +154,33 @@ async function resolveEndpointFromManifest(manifestUrl, fetchImpl) {
|
|
|
104
154
|
throw new Error(`Failed to fetch EvalOps MCP manifest (${response.status} ${response.statusText})`);
|
|
105
155
|
}
|
|
106
156
|
const payload = (await response.json());
|
|
107
|
-
const
|
|
108
|
-
|
|
157
|
+
const payloadRecord = isRecord(payload) ? payload : undefined;
|
|
158
|
+
const protocol = payloadRecord && isRecord(payloadRecord.protocol)
|
|
159
|
+
? payloadRecord.protocol
|
|
109
160
|
: undefined;
|
|
110
161
|
const endpoint = nonEmptyString(protocol?.endpoint);
|
|
111
162
|
if (!endpoint) {
|
|
112
163
|
throw new Error("EvalOps MCP manifest did not include protocol.endpoint");
|
|
113
164
|
}
|
|
114
|
-
|
|
165
|
+
const identity = payloadRecord?.identity;
|
|
166
|
+
const identityBaseUrl = nonEmptyString(isRecord(identity) ? identity.base_url : undefined) ??
|
|
167
|
+
nonEmptyString(isRecord(identity) ? identity.baseUrl : undefined) ??
|
|
168
|
+
nonEmptyString(isRecord(identity) ? identity.url : undefined) ??
|
|
169
|
+
nonEmptyString(payloadRecord?.identity_base_url) ??
|
|
170
|
+
nonEmptyString(payloadRecord?.identityBaseUrl);
|
|
171
|
+
return {
|
|
172
|
+
endpoint: normalizeMcpEndpoint(endpoint),
|
|
173
|
+
identityBaseUrl,
|
|
174
|
+
manifestUrl,
|
|
175
|
+
preferDerivedIdentity: true,
|
|
176
|
+
};
|
|
115
177
|
}
|
|
116
178
|
async function resolveAgentMcpEndpoint(options, deps) {
|
|
117
179
|
if (options.mcpUrl) {
|
|
118
|
-
return {
|
|
180
|
+
return {
|
|
181
|
+
endpoint: normalizeMcpEndpoint(options.mcpUrl),
|
|
182
|
+
preferDerivedIdentity: true,
|
|
183
|
+
};
|
|
119
184
|
}
|
|
120
185
|
if (options.manifestUrl) {
|
|
121
186
|
return resolveEndpointFromManifest(normalizeManifestUrl(options.manifestUrl), deps.fetch);
|
|
@@ -126,7 +191,10 @@ async function resolveAgentMcpEndpoint(options, deps) {
|
|
|
126
191
|
"MAESTRO_EVALOPS_AGENT_MCP_URL",
|
|
127
192
|
]);
|
|
128
193
|
if (configuredMcpUrl) {
|
|
129
|
-
return {
|
|
194
|
+
return {
|
|
195
|
+
endpoint: normalizeMcpEndpoint(configuredMcpUrl),
|
|
196
|
+
preferDerivedIdentity: true,
|
|
197
|
+
};
|
|
130
198
|
}
|
|
131
199
|
const configuredManifestUrl = getEnvValue([
|
|
132
200
|
"MAESTRO_PLATFORM_MCP_MANIFEST_URL",
|
|
@@ -171,19 +239,12 @@ function createDefaultMcpClient(endpoint, bearerToken) {
|
|
|
171
239
|
connect: () => client.connect(transport),
|
|
172
240
|
};
|
|
173
241
|
}
|
|
174
|
-
async function
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
const result = await client.callTool(toolName, args);
|
|
179
|
-
if (result.isError) {
|
|
180
|
-
throw new Error(`${toolName} returned an MCP error`);
|
|
181
|
-
}
|
|
182
|
-
return parseToolOutput(toolName, result);
|
|
183
|
-
}
|
|
184
|
-
finally {
|
|
185
|
-
await client.close().catch(() => undefined);
|
|
242
|
+
async function callConnectedMcpTool(client, toolName, args) {
|
|
243
|
+
const result = await client.callTool(toolName, args);
|
|
244
|
+
if (result.isError) {
|
|
245
|
+
throw new Error(`${toolName} returned an MCP error`);
|
|
186
246
|
}
|
|
247
|
+
return parseToolOutput(toolName, result);
|
|
187
248
|
}
|
|
188
249
|
function parseToolOutput(toolName, result) {
|
|
189
250
|
const structuredContent = isRecord(result.structuredContent)
|
|
@@ -238,27 +299,55 @@ function buildKeyName(options, now) {
|
|
|
238
299
|
.slice(0, 48);
|
|
239
300
|
return `maestro-init-${host || "local"}-${now.toISOString().slice(0, 10)}`;
|
|
240
301
|
}
|
|
241
|
-
async function createAgentAPIKey(options,
|
|
242
|
-
const
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
302
|
+
async function createAgentAPIKey(options, identityBaseUrl, oauthToken, fetchImpl, now) {
|
|
303
|
+
const expiresInDays = positiveInteger(options.expiresInDays);
|
|
304
|
+
const expiresAt = expiresInDays
|
|
305
|
+
? new Date(now.getTime() + expiresInDays * 24 * 60 * 60 * 1000).toISOString()
|
|
306
|
+
: undefined;
|
|
307
|
+
const response = await fetchImpl(`${identityBaseUrl}${PLATFORM_HTTP_ROUTES.identity.apiKeys}`, {
|
|
308
|
+
method: "POST",
|
|
309
|
+
headers: {
|
|
310
|
+
Authorization: `Bearer ${oauthToken}`,
|
|
311
|
+
"Content-Type": "application/json",
|
|
312
|
+
},
|
|
313
|
+
body: JSON.stringify({
|
|
314
|
+
name: buildKeyName(options, now),
|
|
315
|
+
scopes: options.apiKeyScopes ?? DEFAULT_API_KEY_SCOPES,
|
|
316
|
+
...(expiresAt ? { expires_at: expiresAt } : {}),
|
|
317
|
+
}),
|
|
249
318
|
});
|
|
250
|
-
|
|
319
|
+
const output = (await response
|
|
320
|
+
.json()
|
|
321
|
+
.catch(() => ({})));
|
|
322
|
+
if (!response.ok) {
|
|
323
|
+
throw new Error(typeof output.error === "string" && output.error.trim()
|
|
324
|
+
? output.error
|
|
325
|
+
: `EvalOps API key creation failed (${response.status})`);
|
|
326
|
+
}
|
|
327
|
+
const key = isRecord(output.key) ? output.key : undefined;
|
|
328
|
+
const normalized = {
|
|
329
|
+
api_key: nonEmptyString(output.api_key) ?? "",
|
|
330
|
+
expires_at: nonEmptyString(output.expires_at) ?? nonEmptyString(key?.expires_at),
|
|
331
|
+
key_id: nonEmptyString(output.key_id) ?? nonEmptyString(key?.id),
|
|
332
|
+
name: nonEmptyString(output.name) ?? nonEmptyString(key?.name),
|
|
333
|
+
prefix: nonEmptyString(output.prefix) ?? nonEmptyString(key?.prefix),
|
|
334
|
+
scopes: stringArray(output.scopes) ??
|
|
335
|
+
stringArray(key?.scopes) ??
|
|
336
|
+
stringArray(output.scopes_granted),
|
|
337
|
+
};
|
|
338
|
+
if (!nonEmptyString(normalized.api_key)) {
|
|
251
339
|
throw new Error("EvalOps API key creation did not return api_key");
|
|
252
340
|
}
|
|
253
|
-
return
|
|
341
|
+
return normalized;
|
|
254
342
|
}
|
|
255
|
-
async function registerAgent(options,
|
|
256
|
-
const output = await
|
|
343
|
+
async function registerAgent(options, client) {
|
|
344
|
+
const output = await callConnectedMcpTool(client, "evalops_register", {
|
|
257
345
|
agent_type: options.agentType ?? DEFAULT_AGENT_TYPE,
|
|
258
346
|
capabilities: ["maestro:init", "maestro:cli"],
|
|
259
|
-
|
|
347
|
+
...(options.registerScopes?.length
|
|
348
|
+
? { scopes: options.registerScopes }
|
|
349
|
+
: {}),
|
|
260
350
|
surface: options.surface ?? DEFAULT_SURFACE,
|
|
261
|
-
user_token: apiKey,
|
|
262
351
|
...(positiveInteger(options.ttlSeconds)
|
|
263
352
|
? { ttl_seconds: options.ttlSeconds }
|
|
264
353
|
: {}),
|
|
@@ -269,10 +358,91 @@ async function registerAgent(options, endpoint, apiKey, createClient) {
|
|
|
269
358
|
}
|
|
270
359
|
return output;
|
|
271
360
|
}
|
|
361
|
+
async function runGovernedInferenceCheck(client) {
|
|
362
|
+
return callConnectedMcpTool(client, "evalops_check_action", {
|
|
363
|
+
action_type: "llm_gateway.invoke",
|
|
364
|
+
action_payload: "maestro init first governed inference check",
|
|
365
|
+
declared_risk_level: "low",
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
async function loadControlPlaneSummary(client) {
|
|
369
|
+
return callConnectedMcpTool(client, "evalops_control_plane_summary", {});
|
|
370
|
+
}
|
|
272
371
|
function organizationIdFromCredentials(credentials) {
|
|
273
372
|
return (nonEmptyString(credentials?.metadata?.organizationId) ??
|
|
274
373
|
nonEmptyString(credentials?.metadata?.organization_id));
|
|
275
374
|
}
|
|
375
|
+
function authenticatedAsFromCredentials(credentials) {
|
|
376
|
+
const metadata = credentials?.metadata;
|
|
377
|
+
const user = isRecord(metadata?.user) ? metadata.user : undefined;
|
|
378
|
+
return (nonEmptyString(metadata?.email) ??
|
|
379
|
+
nonEmptyString(metadata?.preferred_username) ??
|
|
380
|
+
nonEmptyString(metadata?.preferredUsername) ??
|
|
381
|
+
nonEmptyString(metadata?.user) ??
|
|
382
|
+
nonEmptyString(user?.email) ??
|
|
383
|
+
nonEmptyString(user?.name));
|
|
384
|
+
}
|
|
385
|
+
function consoleUrlFromEndpoint(endpoint) {
|
|
386
|
+
const parsed = new URL(endpoint);
|
|
387
|
+
parsed.pathname = "/overview";
|
|
388
|
+
parsed.search = "";
|
|
389
|
+
parsed.hash = "";
|
|
390
|
+
const env = parsed.hostname === "app.evalops.dev"
|
|
391
|
+
? "production"
|
|
392
|
+
: parsed.hostname === "staging.evalops.dev"
|
|
393
|
+
? "staging"
|
|
394
|
+
: "local";
|
|
395
|
+
parsed.searchParams.set("env", env);
|
|
396
|
+
return parsed.toString();
|
|
397
|
+
}
|
|
398
|
+
function countHighRiskFindings(summary) {
|
|
399
|
+
const findings = Array.isArray(summary.findings) ? summary.findings : [];
|
|
400
|
+
const findingCount = findings.filter((finding) => {
|
|
401
|
+
const record = isRecord(finding) ? finding : undefined;
|
|
402
|
+
const severity = nonEmptyString(record?.severity)?.toLowerCase();
|
|
403
|
+
return severity === "critical" || severity === "high";
|
|
404
|
+
}).length;
|
|
405
|
+
const metricCount = summary.metrics?.high_risk_tools;
|
|
406
|
+
if (typeof metricCount === "number" && Number.isFinite(metricCount)) {
|
|
407
|
+
return Math.max(findingCount, Math.max(0, Math.trunc(metricCount)));
|
|
408
|
+
}
|
|
409
|
+
return findingCount;
|
|
410
|
+
}
|
|
411
|
+
function hasPolicyControl(summary) {
|
|
412
|
+
const approvalRequired = summary.metrics?.approval_required_tools ?? 0;
|
|
413
|
+
if (approvalRequired > 0) {
|
|
414
|
+
return true;
|
|
415
|
+
}
|
|
416
|
+
const controls = Array.isArray(summary.policy_controls)
|
|
417
|
+
? summary.policy_controls
|
|
418
|
+
: [];
|
|
419
|
+
return controls.some((control) => {
|
|
420
|
+
const record = isRecord(control) ? control : undefined;
|
|
421
|
+
return /approval|policy|starter/i.test([
|
|
422
|
+
nonEmptyString(record?.label),
|
|
423
|
+
nonEmptyString(record?.value),
|
|
424
|
+
nonEmptyString(record?.detail),
|
|
425
|
+
]
|
|
426
|
+
.filter(Boolean)
|
|
427
|
+
.join(" "));
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
function hasTraceEvidence(summary) {
|
|
431
|
+
const evidence = Array.isArray(summary.evidence) ? summary.evidence : [];
|
|
432
|
+
return evidence.some((entry) => {
|
|
433
|
+
const record = isRecord(entry) ? entry : undefined;
|
|
434
|
+
return Boolean(nonEmptyString(record?.trace) ??
|
|
435
|
+
nonEmptyString(record?.trace_id) ??
|
|
436
|
+
nonEmptyString(record?.traceId));
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
function governedActionCount(summary) {
|
|
440
|
+
const metricTotal = summary.metrics?.total_tools;
|
|
441
|
+
if (typeof metricTotal === "number" && Number.isFinite(metricTotal)) {
|
|
442
|
+
return Math.max(0, Math.trunc(metricTotal));
|
|
443
|
+
}
|
|
444
|
+
return Array.isArray(summary.tools) ? summary.tools.length : 0;
|
|
445
|
+
}
|
|
276
446
|
function saveBootstrapMetadata(credentials, agentMcp, saveCredentials) {
|
|
277
447
|
saveCredentials("evalops", {
|
|
278
448
|
...credentials,
|
|
@@ -305,13 +475,14 @@ export async function bootstrapEvalOpsAgent(options = {}, dependencies = {}) {
|
|
|
305
475
|
const endpoint = await resolveAgentMcpEndpoint(options, deps);
|
|
306
476
|
const credentialsBeforeKey = deps.loadCredentials("evalops");
|
|
307
477
|
const stored = getStoredAgentMcpMetadata(credentialsBeforeKey);
|
|
478
|
+
const identityBaseUrl = resolveIdentityBaseUrl(credentialsBeforeKey, endpoint);
|
|
308
479
|
const now = deps.now();
|
|
309
480
|
let apiKey = stored && !options.rotateKey ? stored.apiKey : undefined;
|
|
310
481
|
let keyOutput;
|
|
311
482
|
let apiKeyCreated = false;
|
|
312
483
|
if (!apiKey) {
|
|
313
484
|
deps.onStatus({ message: "Creating EvalOps agent API key" });
|
|
314
|
-
keyOutput = await createAgentAPIKey(options,
|
|
485
|
+
keyOutput = await createAgentAPIKey(options, identityBaseUrl, oauthToken, deps.fetch, now);
|
|
315
486
|
apiKey = keyOutput.api_key;
|
|
316
487
|
apiKeyCreated = true;
|
|
317
488
|
}
|
|
@@ -320,8 +491,21 @@ export async function bootstrapEvalOpsAgent(options = {}, dependencies = {}) {
|
|
|
320
491
|
}
|
|
321
492
|
deps.onStatus({ message: "Registering Maestro with EvalOps agent MCP" });
|
|
322
493
|
let registerOutput;
|
|
494
|
+
let client;
|
|
495
|
+
const openAndRegister = async (token) => {
|
|
496
|
+
const mcpClient = deps.createMcpClient(endpoint.endpoint, token);
|
|
497
|
+
await mcpClient.connect();
|
|
498
|
+
try {
|
|
499
|
+
const output = await registerAgent(options, mcpClient);
|
|
500
|
+
return { client: mcpClient, registerOutput: output };
|
|
501
|
+
}
|
|
502
|
+
catch (error) {
|
|
503
|
+
await mcpClient.close().catch(() => undefined);
|
|
504
|
+
throw error;
|
|
505
|
+
}
|
|
506
|
+
};
|
|
323
507
|
try {
|
|
324
|
-
registerOutput = await
|
|
508
|
+
({ client, registerOutput } = await openAndRegister(apiKey));
|
|
325
509
|
}
|
|
326
510
|
catch (error) {
|
|
327
511
|
if (apiKeyCreated) {
|
|
@@ -330,10 +514,35 @@ export async function bootstrapEvalOpsAgent(options = {}, dependencies = {}) {
|
|
|
330
514
|
deps.onStatus({
|
|
331
515
|
message: "Stored EvalOps agent API key failed; rotating and retrying",
|
|
332
516
|
});
|
|
333
|
-
keyOutput = await createAgentAPIKey({ ...options, rotateKey: true },
|
|
517
|
+
keyOutput = await createAgentAPIKey({ ...options, rotateKey: true }, identityBaseUrl, oauthToken, deps.fetch, now);
|
|
334
518
|
apiKey = keyOutput.api_key;
|
|
335
519
|
apiKeyCreated = true;
|
|
336
|
-
registerOutput = await
|
|
520
|
+
({ client, registerOutput } = await openAndRegister(apiKey));
|
|
521
|
+
}
|
|
522
|
+
let governedInferenceCheck = {};
|
|
523
|
+
let controlPlaneSummary = {};
|
|
524
|
+
try {
|
|
525
|
+
deps.onStatus({ message: "Running first governed inference check" });
|
|
526
|
+
governedInferenceCheck = await runGovernedInferenceCheck(client);
|
|
527
|
+
}
|
|
528
|
+
catch (error) {
|
|
529
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
530
|
+
deps.onStatus({
|
|
531
|
+
message: `EvalOps governed inference check unavailable; continuing bootstrap (${reason})`,
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
try {
|
|
535
|
+
deps.onStatus({ message: "Loading EvalOps control-plane proof" });
|
|
536
|
+
controlPlaneSummary = await loadControlPlaneSummary(client);
|
|
537
|
+
}
|
|
538
|
+
catch (error) {
|
|
539
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
540
|
+
deps.onStatus({
|
|
541
|
+
message: `EvalOps control-plane proof unavailable; continuing bootstrap (${reason})`,
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
finally {
|
|
545
|
+
await client.close().catch(() => undefined);
|
|
337
546
|
}
|
|
338
547
|
const credentials = deps.loadCredentials("evalops");
|
|
339
548
|
if (!credentials) {
|
|
@@ -366,15 +575,24 @@ export async function bootstrapEvalOpsAgent(options = {}, dependencies = {}) {
|
|
|
366
575
|
return {
|
|
367
576
|
agentId: registerOutput.agent_id,
|
|
368
577
|
apiKeyCreated,
|
|
578
|
+
approvalPolicyAttached: hasPolicyControl(controlPlaneSummary),
|
|
579
|
+
authenticatedAs: authenticatedAsFromCredentials(credentials),
|
|
580
|
+
consoleUrl: consoleUrlFromEndpoint(endpoint.endpoint),
|
|
369
581
|
endpoint: endpoint.endpoint,
|
|
582
|
+
evidenceEventPublished: (controlPlaneSummary.evidence ?? []).length > 0,
|
|
583
|
+
evidenceEvents: (controlPlaneSummary.evidence ?? []).length,
|
|
584
|
+
governedActionsLoaded: governedActionCount(controlPlaneSummary),
|
|
585
|
+
governedInferenceCheckRan: Boolean(governedInferenceCheck.decision),
|
|
370
586
|
keyPrefix,
|
|
371
587
|
manifestUrl: endpoint.manifestUrl,
|
|
372
588
|
organizationId: organizationIdFromCredentials(credentials),
|
|
373
589
|
registryVisible: registerOutput.registry_visible,
|
|
590
|
+
riskFindings: countHighRiskFindings(controlPlaneSummary),
|
|
374
591
|
runId: registerOutput.run_id,
|
|
375
592
|
scopesGranted: registerOutput.scopes_granted,
|
|
376
593
|
sessionExpiresAt: registerOutput.expires_at,
|
|
377
594
|
stored: true,
|
|
595
|
+
traceIngestionStarted: hasTraceEvidence(controlPlaneSummary),
|
|
378
596
|
};
|
|
379
597
|
}
|
|
380
598
|
//# sourceMappingURL=agent-bootstrap.js.map
|