@datalayer/core 1.2.27 → 1.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/evals/__tests__/external-definition.golden.json +110 -0
- package/lib/api/evals/client.d.ts +13 -1
- package/lib/api/evals/client.js +13 -0
- package/lib/api/evals/types.d.ts +168 -1
- package/lib/api/mcp/agentAccess.d.ts +10 -0
- package/lib/api/mcp/agentAccess.js +28 -0
- package/lib/api/mcp/generated.js +20 -0
- package/lib/api/mcp/index.d.ts +1 -0
- package/lib/api/mcp/index.js +1 -0
- package/lib/components/index.d.ts +1 -0
- package/lib/components/index.js +1 -0
- package/lib/components/mcp/ScopeList.js +10 -1
- package/lib/components/toast/ToastPanel.d.ts +11 -0
- package/lib/components/toast/ToastPanel.js +31 -0
- package/lib/components/toast/index.d.ts +2 -0
- package/lib/components/toast/index.js +6 -0
- package/lib/components/toast/toastQueue.d.ts +33 -0
- package/lib/components/toast/toastQueue.js +47 -0
- package/lib/hooks/useDatalayer.js +12 -16
- package/lib/hooks/useToast.d.ts +6 -0
- package/lib/hooks/useToast.js +40 -1
- package/lib/models/McpAgentAccess.d.ts +95 -0
- package/lib/models/McpAgentAccess.js +5 -0
- package/lib/models/index.d.ts +1 -0
- package/lib/models/index.js +1 -0
- package/lib/otel/views/OtelLive.js +2 -2
- package/lib/otel/views/OtelLiveSpans.js +2 -2
- package/lib/otel/views/OtelLogsList.js +3 -3
- package/lib/otel/views/OtelMetricsChart.d.ts +13 -0
- package/lib/otel/views/OtelMetricsChart.js +109 -26
- package/lib/otel/views/OtelSpanDetail.js +3 -3
- package/lib/otel/views/OtelSqlView.js +4 -4
- package/lib/otel/views/OtelSystemView.js +2 -2
- package/lib/otel/views/OtelTimelineRangeSlider.js +2 -2
- package/lib/otel/views/OtelTracesList.js +2 -2
- package/lib/views/mcp/AuditLog.js +1 -1
- package/lib/views/mcp/ConnectedAgents.d.ts +1 -1
- package/lib/views/mcp/ConnectedAgents.js +13 -10
- package/lib/views/mcp/McpDashboard.d.ts +1 -1
- package/lib/views/mcp/McpDashboard.js +6 -6
- package/lib/views/mcp/McpHome.d.ts +3 -1
- package/lib/views/mcp/McpHome.js +19 -9
- package/lib/views/mcp/Policies.js +1 -1
- package/lib/views/mcp/types.d.ts +8 -4
- package/lib/views/otel/OtelHeader.js +2 -2
- package/package.json +2 -2
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
{
|
|
2
|
+
"external_definition": {
|
|
3
|
+
"adapter_version": "1",
|
|
4
|
+
"capabilities": [
|
|
5
|
+
{
|
|
6
|
+
"code": "",
|
|
7
|
+
"component": "tasks.py@arithmetic:generate",
|
|
8
|
+
"kind": "solver",
|
|
9
|
+
"reason": "",
|
|
10
|
+
"status": "framework-run"
|
|
11
|
+
}
|
|
12
|
+
],
|
|
13
|
+
"configuration": {
|
|
14
|
+
"aggregation": {},
|
|
15
|
+
"missing": [],
|
|
16
|
+
"models": ["bedrock:us.anthropic.claude-haiku-4-5"],
|
|
17
|
+
"scorers": ["includes"],
|
|
18
|
+
"task_args": {},
|
|
19
|
+
"tasks": [
|
|
20
|
+
{
|
|
21
|
+
"entry_point": "tasks.py@arithmetic",
|
|
22
|
+
"scorers": [
|
|
23
|
+
{
|
|
24
|
+
"metrics": ["inspect_ai/accuracy"],
|
|
25
|
+
"name": "includes"
|
|
26
|
+
}
|
|
27
|
+
],
|
|
28
|
+
"solvers": [
|
|
29
|
+
{
|
|
30
|
+
"name": "generate",
|
|
31
|
+
"params": {}
|
|
32
|
+
}
|
|
33
|
+
]
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
"tools": ["add"]
|
|
37
|
+
},
|
|
38
|
+
"content_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
39
|
+
"entry_point": "tasks.py@arithmetic",
|
|
40
|
+
"framework": "inspect-ai",
|
|
41
|
+
"framework_version": "0.3.286",
|
|
42
|
+
"source": {
|
|
43
|
+
"filename": "fixture.tar.gz",
|
|
44
|
+
"kind": "archive",
|
|
45
|
+
"reference": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
"external_plan": {
|
|
49
|
+
"adapter_version": "1",
|
|
50
|
+
"capabilities": [
|
|
51
|
+
{
|
|
52
|
+
"code": "",
|
|
53
|
+
"component": "tasks.py@arithmetic:generate",
|
|
54
|
+
"kind": "solver",
|
|
55
|
+
"reason": "",
|
|
56
|
+
"status": "framework-run"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"code": "",
|
|
60
|
+
"component": "tasks.py@arithmetic:includes",
|
|
61
|
+
"kind": "scorer",
|
|
62
|
+
"reason": "",
|
|
63
|
+
"status": "framework-run"
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"code": "",
|
|
67
|
+
"component": "tasks.py@arithmetic:includes:inspect_ai/accuracy",
|
|
68
|
+
"kind": "metric",
|
|
69
|
+
"reason": "",
|
|
70
|
+
"status": "framework-run"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"code": "",
|
|
74
|
+
"component": "tool:add",
|
|
75
|
+
"kind": "tool",
|
|
76
|
+
"reason": "",
|
|
77
|
+
"status": "framework-run"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"code": "",
|
|
81
|
+
"component": "model:bedrock:us.anthropic.claude-haiku-4-5",
|
|
82
|
+
"kind": "model",
|
|
83
|
+
"reason": "",
|
|
84
|
+
"status": "framework-run"
|
|
85
|
+
}
|
|
86
|
+
],
|
|
87
|
+
"environment": "inspect-ai-env",
|
|
88
|
+
"framework": "inspect-ai",
|
|
89
|
+
"framework_version": "0.3.286",
|
|
90
|
+
"policy": {
|
|
91
|
+
"cpu_seconds": 1800,
|
|
92
|
+
"max_bytes": 1073741824,
|
|
93
|
+
"max_files": 20000,
|
|
94
|
+
"memory_bytes": 8589934592,
|
|
95
|
+
"network": "inference",
|
|
96
|
+
"root_dirs": [],
|
|
97
|
+
"secrets": [],
|
|
98
|
+
"wall_seconds": 1800
|
|
99
|
+
},
|
|
100
|
+
"reproduction": "pip install inspect-ai==0.3.286\nDATALAYER_BASE_URL=<ai-inference>/api/ai-inference/v1 DATALAYER_API_KEY=<your Datalayer token> inspect eval tasks.py@arithmetic --model openai-api/datalayer/bedrock:us.anthropic.claude-haiku-4-5 --log-format json",
|
|
101
|
+
"resolved_configuration": {
|
|
102
|
+
"aggregation": {},
|
|
103
|
+
"approved_tools": ["add"],
|
|
104
|
+
"environment_only": false,
|
|
105
|
+
"max_samples": 1,
|
|
106
|
+
"task_args": {},
|
|
107
|
+
"task_environments": {}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* @module api/evals/client
|
|
10
10
|
*/
|
|
11
11
|
import { type EvalsClientOptions } from './request';
|
|
12
|
-
import type { CaseListResponse, CreateReportRequest, MoveReportRequest, RegenerateReportResponse, ReportListResponse, ReportResponse, ReportsQuery, DecisionListResponse, DecisionRequest, DecisionResponse, DecisionSubject, SharedWithMeResponse, ContinueReportImportResponse, ReportImportListResponse, ReportImportRequest, ReportImportResponse, EvalsPermissionsResponse, EvalsSharingResponse, EvalsSharingUpdate, SharedEvalsRecord, EvalEvaluatorRef, EvalsetVersionListResponse, EvalsetVersionResponse, InvestigationListResponse, InvestigationResponse, InvestigationScope, InvestigationStatus, LexicalReportResponse, ReportDocumentResponse, ResumeSandboxRequest, ResumeSandboxResponse, TaskInvestigationResponse, UpdateInvestigationRequest, EvalsetCategory, ImportEvalsetRequest, ImportEvalsetResponse, CaseResultListResponse, CaseResultResponse, RunNetworkResponse, RunNetworkMessageResponse, LaunchNetworkResponse, CreateLaunchRequest, LaunchCancelResponse, LaunchListResponse, ClaimTrialResponse, LaunchPlanResponse, LaunchResponse, LiveAlertListResponse, SubjectsResponse, ReviewCaseRequest, RunCancelResponse, CaseRequest, CaseResponse, CreateEvalsetRequest, CreateExperimentRequest, CreateLiveEventRequest, CreateRunRequest, EvalKind, EvalRunEnvironment, EvalsetDeleteResponse, EvalsetListResponse, EvalsetResponse, EvalsetPackagePreviewResponse, EvalsetComparisonResponse, SubjectUsageResponse, SandboxProvenanceResponse, BenchmarkComputeResponse, EvalsSearchResponse, EvalsetLeaderboardResponse, EvalsetPublicationResponse, EvalsetPublicationListResponse, ExperimentDeleteResponse, ExperimentListResponse, ExperimentResponse, LiveEventListResponse, LiveEventResponse, LiveTargetDeleteResponse, LiveTargetListResponse, PublicEvalsetDetailsResponse, RunListResponse, RunResponse, SuccessResponse, UpdateEvalsetRequest, UpdateExperimentRequest } from './types';
|
|
12
|
+
import type { AdapterInspectionResponse, EvalAdapterListResponse, InspectExternalSourceRequest, CaseListResponse, CreateReportRequest, MoveReportRequest, RegenerateReportResponse, ReportListResponse, ReportResponse, ReportsQuery, DecisionListResponse, DecisionRequest, DecisionResponse, DecisionSubject, SharedWithMeResponse, ContinueReportImportResponse, ReportImportListResponse, ReportImportRequest, ReportImportResponse, EvalsPermissionsResponse, EvalsSharingResponse, EvalsSharingUpdate, SharedEvalsRecord, EvalEvaluatorRef, EvalsetVersionListResponse, EvalsetVersionResponse, InvestigationListResponse, InvestigationResponse, InvestigationScope, InvestigationStatus, LexicalReportResponse, ReportDocumentResponse, ResumeSandboxRequest, ResumeSandboxResponse, TaskInvestigationResponse, UpdateInvestigationRequest, EvalsetCategory, ImportEvalsetRequest, ImportEvalsetResponse, CaseResultListResponse, CaseResultResponse, RunNetworkResponse, RunNetworkMessageResponse, LaunchNetworkResponse, CreateLaunchRequest, LaunchCancelResponse, LaunchListResponse, ClaimTrialResponse, LaunchPlanResponse, LaunchResponse, LiveAlertListResponse, SubjectsResponse, ReviewCaseRequest, RunCancelResponse, CaseRequest, CaseResponse, CreateEvalsetRequest, CreateExperimentRequest, CreateLiveEventRequest, CreateRunRequest, EvalKind, EvalRunEnvironment, EvalsetDeleteResponse, EvalsetListResponse, EvalsetResponse, EvalsetPackagePreviewResponse, EvalsetComparisonResponse, SubjectUsageResponse, SandboxProvenanceResponse, BenchmarkComputeResponse, EvalsSearchResponse, EvalsetLeaderboardResponse, EvalsetPublicationResponse, EvalsetPublicationListResponse, ExperimentDeleteResponse, ExperimentListResponse, ExperimentResponse, LiveEventListResponse, LiveEventResponse, LiveTargetDeleteResponse, LiveTargetListResponse, PublicEvalsetDetailsResponse, RunListResponse, RunResponse, SuccessResponse, UpdateEvalsetRequest, UpdateExperimentRequest } from './types';
|
|
13
13
|
export type ListEvalsetsQuery = {
|
|
14
14
|
kind?: EvalKind;
|
|
15
15
|
run_environment?: EvalRunEnvironment;
|
|
@@ -19,6 +19,8 @@ export type ListEvalsetsQuery = {
|
|
|
19
19
|
dataset?: string;
|
|
20
20
|
/** …and pinned to this revision of it: one revision is not evidence about another. */
|
|
21
21
|
revision?: string;
|
|
22
|
+
/** Only the benchmarks imported from this framework. */
|
|
23
|
+
framework?: string;
|
|
22
24
|
limit?: number;
|
|
23
25
|
offset?: number;
|
|
24
26
|
};
|
|
@@ -299,6 +301,16 @@ export declare const listLaunches: (options: EvalsClientOptions, query?: ListLau
|
|
|
299
301
|
export declare const getLaunch: (options: EvalsClientOptions, launchId: string) => Promise<LaunchResponse>;
|
|
300
302
|
export declare const cancelLaunch: (options: EvalsClientOptions, launchId: string) => Promise<LaunchCancelResponse>;
|
|
301
303
|
export declare const archiveLaunch: (options: EvalsClientOptions, launchId: string) => Promise<SuccessResponse>;
|
|
304
|
+
/** The external frameworks this deployment offers; none hides them. */
|
|
305
|
+
export declare const listEvalAdapters: (options: EvalsClientOptions) => Promise<EvalAdapterListResponse>;
|
|
306
|
+
/** Keep a framework's archive by its digest and inspect it in a sandbox. */
|
|
307
|
+
export declare const inspectExternalSource: (options: EvalsClientOptions, framework: string, body: InspectExternalSourceRequest) => Promise<AdapterInspectionResponse>;
|
|
308
|
+
/** An inspection under way, or what it found. */
|
|
309
|
+
export declare const getAdapterInspection: (options: EvalsClientOptions, inspectionId: string) => Promise<AdapterInspectionResponse>;
|
|
310
|
+
/** Where an imported benchmark's original source is downloaded from. */
|
|
311
|
+
export declare const externalSourceUrl: (options: EvalsClientOptions, evalsetId: string, version?: number) => string;
|
|
312
|
+
/** Where a task's raw evidence is downloaded from, by the digest it names. */
|
|
313
|
+
export declare const caseArtifactUrl: (options: EvalsClientOptions, runId: string, caseId: string, digest: string) => string;
|
|
302
314
|
/** An evalset from a spec file, the same body the CLI and the action send. */
|
|
303
315
|
export declare const importEvalset: (options: EvalsClientOptions, body: ImportEvalsetRequest) => Promise<ImportEvalsetResponse>;
|
|
304
316
|
export declare const listEvalsetVersions: (options: EvalsClientOptions, evalsetId: string, query?: {
|
package/lib/api/evals/client.js
CHANGED
|
@@ -264,6 +264,19 @@ export const getLaunch = (options, launchId) => evalsRequest(options, `/launches
|
|
|
264
264
|
export const cancelLaunch = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}/cancel`, { method: 'POST' });
|
|
265
265
|
export const archiveLaunch = (options, launchId) => evalsRequest(options, `/launches/${segment(launchId)}/archive`, { method: 'POST' });
|
|
266
266
|
// --- Spec import and definition versions (B2-08, B2-09) --------------------
|
|
267
|
+
// --- External frameworks ---------------------------------------------------
|
|
268
|
+
/** The external frameworks this deployment offers; none hides them. */
|
|
269
|
+
export const listEvalAdapters = (options) => evalsRequest(options, '/adapters');
|
|
270
|
+
/** Keep a framework's archive by its digest and inspect it in a sandbox. */
|
|
271
|
+
export const inspectExternalSource = (options, framework, body) => evalsRequest(options, `/adapters/${segment(framework)}/inspect`, { method: 'POST', body });
|
|
272
|
+
/** An inspection under way, or what it found. */
|
|
273
|
+
export const getAdapterInspection = (options, inspectionId) => evalsRequest(options, `/adapters/inspections/${segment(inspectionId)}`);
|
|
274
|
+
/** Where an imported benchmark's original source is downloaded from. */
|
|
275
|
+
export const externalSourceUrl = (options, evalsetId, version) => evalsUrl(options, `/evalsets/${segment(evalsetId)}/external-source`, {
|
|
276
|
+
version,
|
|
277
|
+
});
|
|
278
|
+
/** Where a task's raw evidence is downloaded from, by the digest it names. */
|
|
279
|
+
export const caseArtifactUrl = (options, runId, caseId, digest) => evalsUrl(options, `/runs/${segment(runId)}/cases/${segment(caseId)}/artifacts/${segment(digest)}`);
|
|
267
280
|
/** An evalset from a spec file, the same body the CLI and the action send. */
|
|
268
281
|
export const importEvalset = (options, body) => evalsRequest(options, '/evalsets/import', {
|
|
269
282
|
method: 'POST',
|
package/lib/api/evals/types.d.ts
CHANGED
|
@@ -40,6 +40,152 @@ export interface EvalCase {
|
|
|
40
40
|
metadata: Record<string, unknown>;
|
|
41
41
|
}
|
|
42
42
|
/** An evalset, the benchmark: versioned cases, evaluators, metadata. */
|
|
43
|
+
/** How an adapter runs a framework's component. */
|
|
44
|
+
export type AdapterCapabilityStatus = 'native' | 'framework-run' | 'unsupported';
|
|
45
|
+
/** One component of an external definition, as its adapter classified it. */
|
|
46
|
+
export interface AdapterCapability {
|
|
47
|
+
component: string;
|
|
48
|
+
kind: string;
|
|
49
|
+
status: AdapterCapabilityStatus;
|
|
50
|
+
/** Why it cannot run, for an `unsupported` one. */
|
|
51
|
+
reason: string;
|
|
52
|
+
/** A stable code to count refusals by. */
|
|
53
|
+
code: string;
|
|
54
|
+
}
|
|
55
|
+
/** The immutable thing a definition was read from. */
|
|
56
|
+
export interface ExternalSourceRef {
|
|
57
|
+
kind: 'archive' | 'git' | 'library' | 'registry';
|
|
58
|
+
/** The sha256 of the archive, a commit, or a pinned package. */
|
|
59
|
+
reference: string;
|
|
60
|
+
filename: string;
|
|
61
|
+
}
|
|
62
|
+
/** What a version and a launch keep of an imported definition. */
|
|
63
|
+
export interface ExternalDefinition {
|
|
64
|
+
framework: string;
|
|
65
|
+
adapter_version: string;
|
|
66
|
+
framework_version: string;
|
|
67
|
+
source: ExternalSourceRef;
|
|
68
|
+
entry_point: string;
|
|
69
|
+
configuration: Record<string, unknown>;
|
|
70
|
+
capabilities: AdapterCapability[];
|
|
71
|
+
/** A digest covering every file of the source. */
|
|
72
|
+
content_digest: string;
|
|
73
|
+
}
|
|
74
|
+
/** An external framework this deployment offers. */
|
|
75
|
+
export interface EvalAdapter {
|
|
76
|
+
framework: string;
|
|
77
|
+
adapter_version: string;
|
|
78
|
+
framework_version: string;
|
|
79
|
+
environment: string;
|
|
80
|
+
}
|
|
81
|
+
export interface EvalAdapterListResponse {
|
|
82
|
+
success: boolean;
|
|
83
|
+
adapters: EvalAdapter[];
|
|
84
|
+
}
|
|
85
|
+
export interface InspectedCase {
|
|
86
|
+
native_id: string;
|
|
87
|
+
name: string;
|
|
88
|
+
input: unknown;
|
|
89
|
+
target: unknown;
|
|
90
|
+
entry_point: string;
|
|
91
|
+
metadata: Record<string, unknown>;
|
|
92
|
+
}
|
|
93
|
+
/** What a source defines, found in a sandbox with none of it run. */
|
|
94
|
+
export interface AdapterInspection {
|
|
95
|
+
framework: string;
|
|
96
|
+
adapter_version: string;
|
|
97
|
+
framework_version: string;
|
|
98
|
+
archive_digest: string;
|
|
99
|
+
content_digest: string;
|
|
100
|
+
name: string;
|
|
101
|
+
description: string;
|
|
102
|
+
entry_points: string[];
|
|
103
|
+
cases: InspectedCase[];
|
|
104
|
+
subjects: Array<Record<string, unknown>>;
|
|
105
|
+
scorers: Array<Record<string, unknown>>;
|
|
106
|
+
models: string[];
|
|
107
|
+
tools: string[];
|
|
108
|
+
secrets: string[];
|
|
109
|
+
network: 'none' | 'inference' | 'boundary';
|
|
110
|
+
environment: Record<string, unknown>;
|
|
111
|
+
capabilities: AdapterCapability[];
|
|
112
|
+
configuration: Record<string, unknown>;
|
|
113
|
+
}
|
|
114
|
+
/** An inspection under way, or its answer. */
|
|
115
|
+
export interface AdapterInspectionRecord {
|
|
116
|
+
id: string;
|
|
117
|
+
owner_uid: string;
|
|
118
|
+
framework: string;
|
|
119
|
+
digest: string;
|
|
120
|
+
filename: string;
|
|
121
|
+
status: 'pending' | 'ready' | 'failed';
|
|
122
|
+
inspection: AdapterInspection | Record<string, never>;
|
|
123
|
+
error: string;
|
|
124
|
+
workflow_uid: string;
|
|
125
|
+
created_at: string | null;
|
|
126
|
+
updated_at: string | null;
|
|
127
|
+
}
|
|
128
|
+
export interface AdapterInspectionResponse {
|
|
129
|
+
success: boolean;
|
|
130
|
+
inspection: AdapterInspectionRecord;
|
|
131
|
+
}
|
|
132
|
+
export interface InspectExternalSourceRequest {
|
|
133
|
+
filename: string;
|
|
134
|
+
content_base64: string;
|
|
135
|
+
options?: Record<string, unknown>;
|
|
136
|
+
}
|
|
137
|
+
/** What was chosen from an inspection's preview. */
|
|
138
|
+
export interface ExternalSelection {
|
|
139
|
+
entry_points?: string[];
|
|
140
|
+
limit?: number;
|
|
141
|
+
arguments?: Record<string, unknown>;
|
|
142
|
+
/** Catalogue models of AI Inference, one experiment each. */
|
|
143
|
+
models?: string[];
|
|
144
|
+
/** The headline score: `{scorer}` or `{kind: 'mean' | 'all'}`, with a `threshold`. */
|
|
145
|
+
aggregation?: Record<string, unknown>;
|
|
146
|
+
name?: string;
|
|
147
|
+
}
|
|
148
|
+
/** How a launch of an imported definition runs. */
|
|
149
|
+
export interface ExternalPlan {
|
|
150
|
+
framework: string;
|
|
151
|
+
adapter_version: string;
|
|
152
|
+
framework_version: string;
|
|
153
|
+
environment: string;
|
|
154
|
+
policy: Record<string, unknown>;
|
|
155
|
+
capabilities: AdapterCapability[];
|
|
156
|
+
resolved_configuration: Record<string, unknown>;
|
|
157
|
+
/** The pinned framework command, with no secret in it. */
|
|
158
|
+
reproduction: string;
|
|
159
|
+
}
|
|
160
|
+
/** What a framework said of one task, beside the normalized fields. */
|
|
161
|
+
export interface ExternalCaseResult {
|
|
162
|
+
framework?: string;
|
|
163
|
+
adapter_version?: string;
|
|
164
|
+
framework_version?: string;
|
|
165
|
+
native_id?: string;
|
|
166
|
+
attempt?: number;
|
|
167
|
+
scores?: Array<{
|
|
168
|
+
name: string;
|
|
169
|
+
value: unknown;
|
|
170
|
+
answer: string;
|
|
171
|
+
explanation: string;
|
|
172
|
+
metadata: Record<string, unknown>;
|
|
173
|
+
}>;
|
|
174
|
+
usage?: Record<string, unknown>;
|
|
175
|
+
details?: Record<string, unknown>;
|
|
176
|
+
/** The task's own environment, when it ran in one: what ran, by digest. */
|
|
177
|
+
environment?: {
|
|
178
|
+
uid?: string;
|
|
179
|
+
image_digest?: string;
|
|
180
|
+
base_image?: string;
|
|
181
|
+
};
|
|
182
|
+
artifacts?: Array<{
|
|
183
|
+
name: string;
|
|
184
|
+
media_type: string;
|
|
185
|
+
digest: string;
|
|
186
|
+
size: number;
|
|
187
|
+
}>;
|
|
188
|
+
}
|
|
43
189
|
export interface Evalset {
|
|
44
190
|
id: string;
|
|
45
191
|
owner_uid: string;
|
|
@@ -49,6 +195,8 @@ export interface Evalset {
|
|
|
49
195
|
kind: EvalKind;
|
|
50
196
|
category: EvalsetCategory | '';
|
|
51
197
|
dataset_ref: DatasetRef | null;
|
|
198
|
+
/** Where an imported definition came from; empty for one made here. */
|
|
199
|
+
external_definition?: ExternalDefinition | Record<string, never>;
|
|
52
200
|
/** Moves on every change to cases, evaluators, schema or category. */
|
|
53
201
|
version: number;
|
|
54
202
|
/** How many launches this benchmark has had; the next is number + 1. */
|
|
@@ -187,6 +335,8 @@ export interface EvalRun {
|
|
|
187
335
|
}
|
|
188
336
|
/** A task result: one case of one run, its own document. */
|
|
189
337
|
export interface EvalTaskResult {
|
|
338
|
+
/** What a framework said of this task, beside the normalized fields. */
|
|
339
|
+
external?: ExternalCaseResult;
|
|
190
340
|
id: string;
|
|
191
341
|
run_id: string;
|
|
192
342
|
launch_id: string;
|
|
@@ -229,6 +379,9 @@ export interface EvalTaskResult {
|
|
|
229
379
|
}
|
|
230
380
|
/** A launch: one submission of a benchmark across its experiments. */
|
|
231
381
|
export interface EvalLaunch {
|
|
382
|
+
/** An imported definition's launch: what it ran from, and how. */
|
|
383
|
+
external_definition?: ExternalDefinition | Record<string, never>;
|
|
384
|
+
external_plan?: ExternalPlan | Record<string, never>;
|
|
232
385
|
id: string;
|
|
233
386
|
owner_uid: string;
|
|
234
387
|
account_uid: string;
|
|
@@ -274,6 +427,9 @@ export interface LaunchPlanProblem {
|
|
|
274
427
|
severity: 'error' | 'warning';
|
|
275
428
|
message: string;
|
|
276
429
|
experiment_id?: string;
|
|
430
|
+
/** The component an imported definition's plan refused. */
|
|
431
|
+
component?: string;
|
|
432
|
+
capability_code?: string;
|
|
277
433
|
}
|
|
278
434
|
/** The plan of a launch before it is made (B2-07). */
|
|
279
435
|
export interface LaunchPlanResponse {
|
|
@@ -288,6 +444,10 @@ export interface LaunchPlanResponse {
|
|
|
288
444
|
* submission.
|
|
289
445
|
*/
|
|
290
446
|
run_mode_constraint: string;
|
|
447
|
+
/** How an imported definition would run; null for one made here. */
|
|
448
|
+
external_plan?: ExternalPlan | null;
|
|
449
|
+
/** Whom an imported definition's model calls are metered under, in a sentence. */
|
|
450
|
+
metering?: string;
|
|
291
451
|
problems: LaunchPlanProblem[];
|
|
292
452
|
estimate: {
|
|
293
453
|
cases: number;
|
|
@@ -1032,12 +1192,19 @@ export interface CaseResultResponse {
|
|
|
1032
1192
|
}
|
|
1033
1193
|
export interface ImportEvalsetRequest {
|
|
1034
1194
|
/** An `*.evalset.json` spec, as the CLI and the action read it. */
|
|
1035
|
-
spec
|
|
1195
|
+
spec?: Record<string, unknown>;
|
|
1196
|
+
/** Or an inspected external source. One of the two. */
|
|
1197
|
+
external?: {
|
|
1198
|
+
inspection_id: string;
|
|
1199
|
+
selection?: ExternalSelection;
|
|
1200
|
+
};
|
|
1036
1201
|
run_environment?: EvalRunEnvironment;
|
|
1037
1202
|
}
|
|
1038
1203
|
export interface ImportEvalsetResponse extends EvalsetResponse {
|
|
1039
1204
|
/** Evaluators the platform cannot run, left out of the evalset. */
|
|
1040
1205
|
unsupported_evaluators: string[];
|
|
1206
|
+
/** An external import's experiments, one per model. */
|
|
1207
|
+
experiments?: EvalExperiment[];
|
|
1041
1208
|
}
|
|
1042
1209
|
export interface EvalsetVersionListResponse {
|
|
1043
1210
|
success: boolean;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { McpAgentReach, McpAgentToolsets } from '../../models/McpAgentAccess';
|
|
2
|
+
export type { McpAgentNamed, McpAgentReach, McpAgentTool, McpAgentToolset, McpAgentToolsets, McpReachRow, McpSourceApprovalPolicy, McpToolsetHealth, McpToolsetHealthState, McpToolsetSelected, } from '../../models/McpAgentAccess';
|
|
3
|
+
export interface McpAgentAccessFilters {
|
|
4
|
+
/** One of the caller's connected agents, by its uid. */
|
|
5
|
+
grant?: string;
|
|
6
|
+
}
|
|
7
|
+
/** The toolsets an agent may execute: their tools, approval and health. */
|
|
8
|
+
export declare const getAgentToolsets: (token: string, filters?: McpAgentAccessFilters, baseUrl?: string) => Promise<McpAgentToolsets>;
|
|
9
|
+
/** The notebooks and sources an agent can reach, and how far. */
|
|
10
|
+
export declare const getAgentReach: (token: string, filters?: McpAgentAccessFilters, baseUrl?: string) => Promise<McpAgentReach>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2023-2026 Datalayer, Inc.
|
|
3
|
+
* Distributed under the terms of the Modified BSD License.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* What one agent may execute and reach.
|
|
7
|
+
*
|
|
8
|
+
* Both routes take `grant`, the uid of one of the caller's connected agents;
|
|
9
|
+
* without it the caller is asked about itself, which is how an agent holds
|
|
10
|
+
* these answers against its own token.
|
|
11
|
+
*
|
|
12
|
+
* @module api/mcp/agentAccess
|
|
13
|
+
*/
|
|
14
|
+
import { DEFAULT_SERVICE_URLS } from '../constants.js';
|
|
15
|
+
import { requestDatalayerAPI } from '../DatalayerApi.js';
|
|
16
|
+
import { fromWire, mcpUrl } from './gateway.js';
|
|
17
|
+
/** The toolsets an agent may execute: their tools, approval and health. */
|
|
18
|
+
export const getAgentToolsets = async (token, filters = {}, baseUrl = DEFAULT_SERVICE_URLS.JUPYTER_MCP_SERVER) => fromWire(await requestDatalayerAPI({
|
|
19
|
+
url: mcpUrl(baseUrl, '/toolsets', { ...filters }),
|
|
20
|
+
method: 'GET',
|
|
21
|
+
token,
|
|
22
|
+
}));
|
|
23
|
+
/** The notebooks and sources an agent can reach, and how far. */
|
|
24
|
+
export const getAgentReach = async (token, filters = {}, baseUrl = DEFAULT_SERVICE_URLS.JUPYTER_MCP_SERVER) => fromWire(await requestDatalayerAPI({
|
|
25
|
+
url: mcpUrl(baseUrl, '/reach', { ...filters }),
|
|
26
|
+
method: 'GET',
|
|
27
|
+
token,
|
|
28
|
+
}));
|
package/lib/api/mcp/generated.js
CHANGED
|
@@ -71,6 +71,7 @@ export const MCP_GATEWAY_ROUTES = [
|
|
|
71
71
|
{ name: 'team', in: 'query' },
|
|
72
72
|
{ name: 'user', in: 'query' },
|
|
73
73
|
{ name: 'client', in: 'query' },
|
|
74
|
+
{ name: 'credential', in: 'query' },
|
|
74
75
|
{ name: 'agent', in: 'query' },
|
|
75
76
|
{ name: 'tool', in: 'query' },
|
|
76
77
|
{ name: 'method', in: 'query' },
|
|
@@ -79,6 +80,7 @@ export const MCP_GATEWAY_ROUTES = [
|
|
|
79
80
|
{ name: 'notebook', in: 'query' },
|
|
80
81
|
{ name: 'source', in: 'query' },
|
|
81
82
|
{ name: 'task', in: 'query' },
|
|
83
|
+
{ name: 'trace_id', in: 'query' },
|
|
82
84
|
{ name: 'since', in: 'query' },
|
|
83
85
|
{ name: 'until', in: 'query' },
|
|
84
86
|
{ name: 'cursor', in: 'query' },
|
|
@@ -156,6 +158,12 @@ export const MCP_GATEWAY_ROUTES = [
|
|
|
156
158
|
{ name: 'team', in: 'query' },
|
|
157
159
|
],
|
|
158
160
|
},
|
|
161
|
+
{
|
|
162
|
+
method: 'POST',
|
|
163
|
+
path: '/api/mcp/v1/pap/checks',
|
|
164
|
+
operationId: 'pap_signed_checks_endpoint_api_mcp_v1_pap_checks_post',
|
|
165
|
+
parameters: [],
|
|
166
|
+
},
|
|
159
167
|
{
|
|
160
168
|
method: 'GET',
|
|
161
169
|
path: '/api/mcp/v1/policy',
|
|
@@ -166,6 +174,12 @@ export const MCP_GATEWAY_ROUTES = [
|
|
|
166
174
|
{ name: 'team', in: 'query' },
|
|
167
175
|
],
|
|
168
176
|
},
|
|
177
|
+
{
|
|
178
|
+
method: 'GET',
|
|
179
|
+
path: '/api/mcp/v1/reach',
|
|
180
|
+
operationId: 'agent_reach_endpoint_api_mcp_v1_reach_get',
|
|
181
|
+
parameters: [{ name: 'grant', in: 'query' }],
|
|
182
|
+
},
|
|
169
183
|
{
|
|
170
184
|
method: 'GET',
|
|
171
185
|
path: '/api/mcp/v1/tasks',
|
|
@@ -204,6 +218,12 @@ export const MCP_GATEWAY_ROUTES = [
|
|
|
204
218
|
operationId: 'answer_task_endpoint_api_mcp_v1_tasks__task_uid__input_post',
|
|
205
219
|
parameters: [{ name: 'task_uid', in: 'path' }],
|
|
206
220
|
},
|
|
221
|
+
{
|
|
222
|
+
method: 'GET',
|
|
223
|
+
path: '/api/mcp/v1/toolsets',
|
|
224
|
+
operationId: 'agent_toolsets_endpoint_api_mcp_v1_toolsets_get',
|
|
225
|
+
parameters: [{ name: 'grant', in: 'query' }],
|
|
226
|
+
},
|
|
207
227
|
{
|
|
208
228
|
method: 'GET',
|
|
209
229
|
path: '/api/mcp/version',
|
package/lib/api/mcp/index.d.ts
CHANGED
package/lib/api/mcp/index.js
CHANGED
|
@@ -17,6 +17,7 @@ export * from './generated.js';
|
|
|
17
17
|
export * from './gateway.js';
|
|
18
18
|
export * from './tasks.js';
|
|
19
19
|
export * from './bindings.js';
|
|
20
|
+
export * from './agentAccess.js';
|
|
20
21
|
export * from './audit.js';
|
|
21
22
|
export * from './activity.js';
|
|
22
23
|
export * from './organizations.js';
|
package/lib/components/index.js
CHANGED
|
@@ -9,6 +9,15 @@ export const ScopeList = ({ scopes, max = 3 }) => {
|
|
|
9
9
|
}
|
|
10
10
|
const shown = scopes.slice(0, max);
|
|
11
11
|
const rest = scopes.length - shown.length;
|
|
12
|
-
return (_jsxs(Box, { display: "flex", gap: 1, flexWrap: "wrap", alignItems: "center",
|
|
12
|
+
return (_jsxs(Box, { display: "flex", gap: 1, flexWrap: "wrap", alignItems: "center", minWidth: 0, maxWidth: "100%", children: [shown.map(scope => (
|
|
13
|
+
// A long scope is cut with an ellipsis and said in full on hover,
|
|
14
|
+
// never drawn past its cell into the next column.
|
|
15
|
+
_jsx(Label, { size: "small", variant: "secondary", title: scope, sx: {
|
|
16
|
+
maxWidth: '100%',
|
|
17
|
+
overflow: 'hidden',
|
|
18
|
+
textOverflow: 'ellipsis',
|
|
19
|
+
whiteSpace: 'nowrap',
|
|
20
|
+
display: 'inline-block',
|
|
21
|
+
}, children: scope }, scope))), rest > 0 && (_jsxs(Text, { as: "span", title: scopes.join('\n'), "aria-label": `and ${rest} more: ${scopes.slice(max).join(', ')}`, sx: { fontSize: 0, color: 'fg.muted' }, children: ["+", rest] }))] }));
|
|
13
22
|
};
|
|
14
23
|
export default ScopeList;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where `useToast` shows its toasts outside JupyterLab: primer-addons'
|
|
3
|
+
* sliding panel, one toast at a time, the next once it is gone. Mount it
|
|
4
|
+
* once, inside the application's theme. JupyterLab keeps its own
|
|
5
|
+
* notifications.
|
|
6
|
+
*
|
|
7
|
+
* @module components/toast/ToastPanel
|
|
8
|
+
*/
|
|
9
|
+
import type { JSX } from 'react';
|
|
10
|
+
export declare const ToastPanel: () => JSX.Element;
|
|
11
|
+
export default ToastPanel;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import { Fragment, useSyncExternalStore } from 'react';
|
|
3
|
+
import { Button } from '@primer/react';
|
|
4
|
+
import { Box, SlidingPanel } from '@datalayer/primer-addons';
|
|
5
|
+
import { toastQueue } from './toastQueue.js';
|
|
6
|
+
const BUTTON_VARIANTS = {
|
|
7
|
+
accent: 'primary',
|
|
8
|
+
link: 'invisible',
|
|
9
|
+
warn: 'danger',
|
|
10
|
+
default: 'default',
|
|
11
|
+
};
|
|
12
|
+
export const ToastPanel = () => {
|
|
13
|
+
const toasts = useSyncExternalStore(toastQueue.subscribe, toastQueue.snapshot, toastQueue.snapshot);
|
|
14
|
+
const shown = toasts[0];
|
|
15
|
+
const close = () => {
|
|
16
|
+
if (shown) {
|
|
17
|
+
toastQueue.dismiss(shown.id);
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
return (_jsx(SlidingPanel
|
|
21
|
+
// A new toast is a new panel: it slides in, and its own clock runs.
|
|
22
|
+
, { isOpen: Boolean(shown),
|
|
23
|
+
// A toast of several lines (a failed request's message and exception) says each on its own line.
|
|
24
|
+
message: (shown?.message ?? '').split('\n').map((line, index) => (_jsxs(Fragment, { children: [index > 0 ? _jsx("br", {}) : null, line] }, index))), variant: shown?.variant ?? 'default', position: "north", durationMs: shown && shown.autoClose !== false ? shown.autoClose : undefined, onDismiss: close, details: shown?.actions?.length ? (_jsx(Box, { display: "flex", gap: 2, flexWrap: "wrap", mt: 2, children: shown.actions.map(action => (_jsx(Button, { size: "small", title: action.caption ?? action.label, variant: BUTTON_VARIANTS[action.displayType ?? 'default'], onClick: (event) => {
|
|
25
|
+
action.callback(event.nativeEvent);
|
|
26
|
+
if (!event.nativeEvent.defaultPrevented) {
|
|
27
|
+
close();
|
|
28
|
+
}
|
|
29
|
+
}, children: action.label }, action.label))) })) : undefined }, shown?.id ?? 'none'));
|
|
30
|
+
};
|
|
31
|
+
export default ToastPanel;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The toasts waiting to be shown outside JupyterLab, in the order they were
|
|
3
|
+
* asked for: `useToast` puts them here, `ToastPanel` shows the first in a
|
|
4
|
+
* sliding panel, and taking it away shows the next.
|
|
5
|
+
*
|
|
6
|
+
* @module components/toast/toastQueue
|
|
7
|
+
*/
|
|
8
|
+
export type ToastVariant = 'info' | 'success' | 'warning' | 'error';
|
|
9
|
+
/** A button on a toast, as JupyterLab's notifications name one. */
|
|
10
|
+
export type ToastAction = {
|
|
11
|
+
label: string;
|
|
12
|
+
caption?: string;
|
|
13
|
+
displayType?: 'default' | 'accent' | 'warn' | 'link';
|
|
14
|
+
callback: (event: MouseEvent) => void;
|
|
15
|
+
};
|
|
16
|
+
export type QueuedToast = {
|
|
17
|
+
id: string;
|
|
18
|
+
variant: ToastVariant;
|
|
19
|
+
message: string;
|
|
20
|
+
actions?: ToastAction[];
|
|
21
|
+
/** Milliseconds before it goes on its own; `false` stays until dismissed. */
|
|
22
|
+
autoClose: number | false;
|
|
23
|
+
};
|
|
24
|
+
/** Info and success go on their own after five seconds; a warning or an error stays. */
|
|
25
|
+
export declare const defaultAutoClose: (variant: ToastVariant) => number | false;
|
|
26
|
+
export declare const toastQueue: {
|
|
27
|
+
subscribe(listener: () => void): () => void;
|
|
28
|
+
snapshot(): readonly QueuedToast[];
|
|
29
|
+
push(toast: Omit<QueuedToast, "id">): string;
|
|
30
|
+
update(id: string, patch: Partial<Omit<QueuedToast, "id">>): void;
|
|
31
|
+
/** Take one toast away, or every toast without an id. */
|
|
32
|
+
dismiss(id?: string): void;
|
|
33
|
+
};
|