@dalmia/calibrate-mcp 0.0.87 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mcp-server.js +408 -88
- package/bin/mcp-server.js.map +23 -15
- package/esm/funcs/{workspaceCurrent.d.ts → organizationsCurrent.d.ts} +2 -2
- package/esm/funcs/organizationsCurrent.d.ts.map +1 -0
- package/esm/funcs/{workspaceCurrent.js → organizationsCurrent.js} +2 -2
- package/esm/funcs/organizationsCurrent.js.map +1 -0
- package/esm/funcs/tracesList.d.ts +16 -0
- package/esm/funcs/tracesList.d.ts.map +1 -0
- package/esm/funcs/tracesList.js +102 -0
- package/esm/funcs/tracesList.js.map +1 -0
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +2 -2
- package/esm/lib/config.js +2 -2
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.d.ts.map +1 -1
- package/esm/mcp-server/server.js +5 -3
- package/esm/mcp-server/server.js.map +1 -1
- package/esm/mcp-server/tools/{workspaceCurrent.d.ts → organizationsCurrent.d.ts} +2 -2
- package/esm/mcp-server/tools/organizationsCurrent.d.ts.map +1 -0
- package/esm/mcp-server/tools/{workspaceCurrent.js → organizationsCurrent.js} +4 -4
- package/esm/mcp-server/tools/organizationsCurrent.js.map +1 -0
- package/esm/mcp-server/tools/tracesList.d.ts +7 -0
- package/esm/mcp-server/tools/tracesList.d.ts.map +1 -0
- package/esm/mcp-server/tools/tracesList.js +40 -0
- package/esm/mcp-server/tools/tracesList.js.map +1 -0
- package/esm/models/listtracesendpointtracesgetop.d.ts +55 -0
- package/esm/models/listtracesendpointtracesgetop.d.ts.map +1 -0
- package/esm/models/listtracesendpointtracesgetop.js +52 -0
- package/esm/models/listtracesendpointtracesgetop.js.map +1 -0
- package/esm/models/traceevalrunstatus.d.ts +18 -0
- package/esm/models/traceevalrunstatus.d.ts.map +1 -0
- package/esm/models/traceevalrunstatus.js +19 -0
- package/esm/models/traceevalrunstatus.js.map +1 -0
- package/esm/models/tracelistresponse.d.ts +15 -0
- package/esm/models/tracelistresponse.d.ts.map +1 -0
- package/esm/models/tracelistresponse.js +16 -0
- package/esm/models/tracelistresponse.js.map +1 -0
- package/esm/models/tracescoreaverage.d.ts +38 -0
- package/esm/models/tracescoreaverage.d.ts.map +1 -0
- package/esm/models/tracescoreaverage.js +31 -0
- package/esm/models/tracescoreaverage.js.map +1 -0
- package/esm/models/tracescoreresult.d.ts +58 -0
- package/esm/models/tracescoreresult.d.ts.map +1 -0
- package/esm/models/tracescoreresult.js +50 -0
- package/esm/models/tracescoreresult.js.map +1 -0
- package/esm/models/tracesummary.d.ts +35 -0
- package/esm/models/tracesummary.d.ts.map +1 -0
- package/esm/models/tracesummary.js +42 -0
- package/esm/models/tracesummary.js.map +1 -0
- package/esm/tool-names.d.ts.map +1 -1
- package/esm/tool-names.js +4 -0
- package/esm/tool-names.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/{workspaceCurrent.ts → organizationsCurrent.ts} +1 -1
- package/src/funcs/tracesList.ts +165 -0
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +2 -2
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +5 -3
- package/src/mcp-server/tools/{workspaceCurrent.ts → organizationsCurrent.ts} +3 -3
- package/src/mcp-server/tools/tracesList.ts +48 -0
- package/src/models/listtracesendpointtracesgetop.ts +129 -0
- package/src/models/traceevalrunstatus.ts +23 -0
- package/src/models/tracelistresponse.ts +39 -0
- package/src/models/tracescoreaverage.ts +72 -0
- package/src/models/tracescoreresult.ts +107 -0
- package/src/models/tracesummary.ts +112 -0
- package/src/tool-names.ts +4 -0
- package/esm/funcs/workspaceCurrent.d.ts.map +0 -1
- package/esm/funcs/workspaceCurrent.js.map +0 -1
- package/esm/mcp-server/tools/workspaceCurrent.d.ts.map +0 -1
- package/esm/mcp-server/tools/workspaceCurrent.js.map +0 -1
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { tracesList } from "../../funcs/tracesList.js";
|
|
6
|
+
import { ListTracesEndpointTracesGetRequest$zodSchema } from "../../models/listtracesendpointtracesgetop.js";
|
|
7
|
+
import { formatResult, ToolDefinition } from "../tools.js";
|
|
8
|
+
|
|
9
|
+
const args = {
|
|
10
|
+
request: ListTracesEndpointTracesGetRequest$zodSchema.optional(),
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export const tool$tracesList: ToolDefinition<typeof args> = {
|
|
14
|
+
name: "list-traces",
|
|
15
|
+
description: `List the traces in the caller's org, newest first, one page at a
|
|
16
|
+
time. Each trace carries a short preview of its last user message
|
|
17
|
+
and reply, its tool calls and labels, and the scores from its
|
|
18
|
+
latest scoring run with each evaluator's reasoning and evaluator
|
|
19
|
+
version. Filter by agent, label, creation time, text, or score.
|
|
20
|
+
Set include_content to also get each trace's full input, output
|
|
21
|
+
and metadata.
|
|
22
|
+
`,
|
|
23
|
+
scopes: ["read"],
|
|
24
|
+
annotations: {
|
|
25
|
+
"title": "List traces",
|
|
26
|
+
"destructiveHint": false,
|
|
27
|
+
"idempotentHint": false,
|
|
28
|
+
"openWorldHint": true,
|
|
29
|
+
"readOnlyHint": true,
|
|
30
|
+
},
|
|
31
|
+
args,
|
|
32
|
+
tool: async (client, args, ctx) => {
|
|
33
|
+
const [result] = await tracesList(
|
|
34
|
+
client,
|
|
35
|
+
args.request,
|
|
36
|
+
{ fetchOptions: { signal: ctx.signal } },
|
|
37
|
+
).$inspect();
|
|
38
|
+
|
|
39
|
+
if (!result.ok) {
|
|
40
|
+
return {
|
|
41
|
+
content: [{ type: "text", text: result.error.message }],
|
|
42
|
+
isError: true,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return formatResult(result.value);
|
|
47
|
+
},
|
|
48
|
+
};
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod";
|
|
6
|
+
import { ClosedEnum } from "../types/enums.js";
|
|
7
|
+
import {
|
|
8
|
+
HTTPValidationError,
|
|
9
|
+
HTTPValidationError$zodSchema,
|
|
10
|
+
} from "./httpvalidationerror.js";
|
|
11
|
+
import {
|
|
12
|
+
TraceListResponse,
|
|
13
|
+
TraceListResponse$zodSchema,
|
|
14
|
+
} from "./tracelistresponse.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Return only traces whose output is of this kind. `response` covers every trace carrying a reply, including one that also issued tool calls. `tool_call` covers traces that only issued tool calls
|
|
18
|
+
*/
|
|
19
|
+
export const ListTracesEndpointTracesGetOutputType = {
|
|
20
|
+
Response: "response",
|
|
21
|
+
ToolCall: "tool_call",
|
|
22
|
+
} as const;
|
|
23
|
+
/**
|
|
24
|
+
* Return only traces whose output is of this kind. `response` covers every trace carrying a reply, including one that also issued tool calls. `tool_call` covers traces that only issued tool calls
|
|
25
|
+
*/
|
|
26
|
+
export type ListTracesEndpointTracesGetOutputType = ClosedEnum<
|
|
27
|
+
typeof ListTracesEndpointTracesGetOutputType
|
|
28
|
+
>;
|
|
29
|
+
|
|
30
|
+
export const ListTracesEndpointTracesGetOutputType$zodSchema = z.enum([
|
|
31
|
+
"response",
|
|
32
|
+
"tool_call",
|
|
33
|
+
]).describe(
|
|
34
|
+
"Return only traces whose output is of this kind. `response` covers every trace carrying a reply, including one that also issued tool calls. `tool_call` covers traces that only issued tool calls",
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Whether `sort_by_evaluator` puts the highest scores first or the lowest. Ignored without it
|
|
39
|
+
*/
|
|
40
|
+
export const SortOrder = {
|
|
41
|
+
Asc: "asc",
|
|
42
|
+
Desc: "desc",
|
|
43
|
+
} as const;
|
|
44
|
+
/**
|
|
45
|
+
* Whether `sort_by_evaluator` puts the highest scores first or the lowest. Ignored without it
|
|
46
|
+
*/
|
|
47
|
+
export type SortOrder = ClosedEnum<typeof SortOrder>;
|
|
48
|
+
|
|
49
|
+
export const SortOrder$zodSchema = z.enum([
|
|
50
|
+
"asc",
|
|
51
|
+
"desc",
|
|
52
|
+
]).describe(
|
|
53
|
+
"Whether `sort_by_evaluator` puts the highest scores first or the lowest. Ignored without it",
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
export type ListTracesEndpointTracesGetRequest = {
|
|
57
|
+
agent_id?: string | null | undefined;
|
|
58
|
+
conversation_id?: string | null | undefined;
|
|
59
|
+
q?: string | null | undefined;
|
|
60
|
+
output_type?: ListTracesEndpointTracesGetOutputType | null | undefined;
|
|
61
|
+
labels?: Array<string> | null | undefined;
|
|
62
|
+
score?: Array<string> | null | undefined;
|
|
63
|
+
created_after?: string | null | undefined;
|
|
64
|
+
created_before?: string | null | undefined;
|
|
65
|
+
sort_by_evaluator?: string | null | undefined;
|
|
66
|
+
sort_order?: SortOrder | undefined;
|
|
67
|
+
include_content?: boolean | undefined;
|
|
68
|
+
include_score_averages?: boolean | undefined;
|
|
69
|
+
limit?: number | undefined;
|
|
70
|
+
offset?: number | undefined;
|
|
71
|
+
xAPIKey?: string | null | undefined;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
export const ListTracesEndpointTracesGetRequest$zodSchema: z.ZodType<
|
|
75
|
+
ListTracesEndpointTracesGetRequest
|
|
76
|
+
> = z.object({
|
|
77
|
+
agent_id: z.string().describe("Return only traces from this agent").nullable()
|
|
78
|
+
.optional(),
|
|
79
|
+
conversation_id: z.string().describe(
|
|
80
|
+
"Return only the turns of this conversation. Requires `agent_id`",
|
|
81
|
+
).nullable().optional(),
|
|
82
|
+
created_after: z.iso.datetime({ offset: true }).describe(
|
|
83
|
+
"Return only traces created at or after this time (ISO 8601). A time with no timezone is read as UTC",
|
|
84
|
+
).nullable().optional(),
|
|
85
|
+
created_before: z.iso.datetime({ offset: true }).describe(
|
|
86
|
+
"Return only traces created before this time (ISO 8601). A time with no timezone is read as UTC",
|
|
87
|
+
).nullable().optional(),
|
|
88
|
+
include_content: z.boolean().default(false).describe(
|
|
89
|
+
"Also return each trace's full input, output and metadata. Use `limit` and `offset` to page through them",
|
|
90
|
+
),
|
|
91
|
+
include_score_averages: z.boolean().default(false).describe(
|
|
92
|
+
"Also return each evaluator's mean score over every matching trace, not just this page. Costs a pass over those traces, so ask for it when the filters change rather than on every page",
|
|
93
|
+
),
|
|
94
|
+
labels: z.array(z.string()).describe(
|
|
95
|
+
"Return only traces carrying at least one of these labels. Repeat the parameter for each label",
|
|
96
|
+
).nullable().optional(),
|
|
97
|
+
limit: z.int().default(50).describe("Maximum number of items to return"),
|
|
98
|
+
offset: z.int().default(0).describe(
|
|
99
|
+
"Number of items to skip before returning results",
|
|
100
|
+
),
|
|
101
|
+
output_type: ListTracesEndpointTracesGetOutputType$zodSchema.nullable()
|
|
102
|
+
.optional().describe(
|
|
103
|
+
"Return only traces whose output is of this kind. `response` covers every trace carrying a reply, including one that also issued tool calls. `tool_call` covers traces that only issued tool calls",
|
|
104
|
+
),
|
|
105
|
+
q: z.string().describe(
|
|
106
|
+
"Return only traces containing this text in their message ID, conversation ID, conversation history, output, or metadata",
|
|
107
|
+
).nullable().optional(),
|
|
108
|
+
score: z.array(z.string()).describe(
|
|
109
|
+
"Return only traces whose latest scoring run matches this condition. Each is written as `<evaluator ID>:<condition>`, where the condition is `passed`, `failed`, or one of `=`, `!=`, `<`, `<=`, `>`, `>=` followed by a number, such as `>=4`. Repeat the parameter to require every condition at once",
|
|
110
|
+
).nullable().optional(),
|
|
111
|
+
sort_by_evaluator: z.string().describe(
|
|
112
|
+
"Order traces by this evaluator's score on each trace's latest scoring run. Traces it has not scored come last whichever direction you pick",
|
|
113
|
+
).nullable().optional(),
|
|
114
|
+
sort_order: SortOrder$zodSchema.default("desc").describe(
|
|
115
|
+
"Whether `sort_by_evaluator` puts the highest scores first or the lowest. Ignored without it",
|
|
116
|
+
),
|
|
117
|
+
xAPIKey: z.string().nullable().optional(),
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
export type ListTracesEndpointTracesGetResponse =
|
|
121
|
+
| TraceListResponse
|
|
122
|
+
| HTTPValidationError;
|
|
123
|
+
|
|
124
|
+
export const ListTracesEndpointTracesGetResponse$zodSchema: z.ZodType<
|
|
125
|
+
ListTracesEndpointTracesGetResponse
|
|
126
|
+
> = z.union([
|
|
127
|
+
TraceListResponse$zodSchema,
|
|
128
|
+
HTTPValidationError$zodSchema,
|
|
129
|
+
]);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod";
|
|
6
|
+
import { ClosedEnum } from "../types/enums.js";
|
|
7
|
+
|
|
8
|
+
export const TraceEvalRunStatus = {
|
|
9
|
+
Pending: "pending",
|
|
10
|
+
Processing: "processing",
|
|
11
|
+
Completed: "completed",
|
|
12
|
+
Failed: "failed",
|
|
13
|
+
Skipped: "skipped",
|
|
14
|
+
} as const;
|
|
15
|
+
export type TraceEvalRunStatus = ClosedEnum<typeof TraceEvalRunStatus>;
|
|
16
|
+
|
|
17
|
+
export const TraceEvalRunStatus$zodSchema = z.enum([
|
|
18
|
+
"pending",
|
|
19
|
+
"processing",
|
|
20
|
+
"completed",
|
|
21
|
+
"failed",
|
|
22
|
+
"skipped",
|
|
23
|
+
]);
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod";
|
|
6
|
+
import {
|
|
7
|
+
TraceScoreAverage,
|
|
8
|
+
TraceScoreAverage$zodSchema,
|
|
9
|
+
} from "./tracescoreaverage.js";
|
|
10
|
+
import { TraceSummary, TraceSummary$zodSchema } from "./tracesummary.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The page, plus the averages when `include_score_averages` asked for them.
|
|
14
|
+
*/
|
|
15
|
+
export type TraceListResponse = {
|
|
16
|
+
items: Array<TraceSummary>;
|
|
17
|
+
total: number;
|
|
18
|
+
limit?: number | null | undefined;
|
|
19
|
+
offset?: number | undefined;
|
|
20
|
+
score_averages?: Array<TraceScoreAverage> | null | undefined;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export const TraceListResponse$zodSchema: z.ZodType<TraceListResponse> = z
|
|
24
|
+
.object({
|
|
25
|
+
items: z.array(TraceSummary$zodSchema).describe("The page of results"),
|
|
26
|
+
limit: z.int().nullable().optional().describe(
|
|
27
|
+
"The applied `limit` (null when unbounded)",
|
|
28
|
+
),
|
|
29
|
+
offset: z.int().default(0).describe("The applied `offset`"),
|
|
30
|
+
score_averages: z.array(TraceScoreAverage$zodSchema).nullable().optional()
|
|
31
|
+
.describe(
|
|
32
|
+
"One entry per evaluator that scored at least one matching trace, averaged over every matching trace rather than this page. Ask for it with `include_score_averages`",
|
|
33
|
+
),
|
|
34
|
+
total: z.int().describe(
|
|
35
|
+
"Total number of items matching the query, before pagination",
|
|
36
|
+
),
|
|
37
|
+
}).describe(
|
|
38
|
+
"The page, plus the averages when `include_score_averages` asked for them.",
|
|
39
|
+
);
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod";
|
|
6
|
+
import { ClosedEnum } from "../types/enums.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* How the evaluator scores:
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
*
|
|
13
|
+
* - `binary`: pass or fail
|
|
14
|
+
* - `rating`: a numeric score, using the scale in `output_config`
|
|
15
|
+
*/
|
|
16
|
+
export const TraceScoreAverageOutputType = {
|
|
17
|
+
Binary: "binary",
|
|
18
|
+
Rating: "rating",
|
|
19
|
+
} as const;
|
|
20
|
+
/**
|
|
21
|
+
* How the evaluator scores:
|
|
22
|
+
*
|
|
23
|
+
* @remarks
|
|
24
|
+
*
|
|
25
|
+
* - `binary`: pass or fail
|
|
26
|
+
* - `rating`: a numeric score, using the scale in `output_config`
|
|
27
|
+
*/
|
|
28
|
+
export type TraceScoreAverageOutputType = ClosedEnum<
|
|
29
|
+
typeof TraceScoreAverageOutputType
|
|
30
|
+
>;
|
|
31
|
+
|
|
32
|
+
export const TraceScoreAverageOutputType$zodSchema = z.enum([
|
|
33
|
+
"binary",
|
|
34
|
+
"rating",
|
|
35
|
+
]).describe(
|
|
36
|
+
"How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
export type TraceScoreAverage = {
|
|
40
|
+
evaluator_uuid: string;
|
|
41
|
+
name: string;
|
|
42
|
+
output_type: TraceScoreAverageOutputType;
|
|
43
|
+
traces_scored: number;
|
|
44
|
+
average: number;
|
|
45
|
+
scale_min?: number | null | undefined;
|
|
46
|
+
scale_max?: number | null | undefined;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export const TraceScoreAverage$zodSchema: z.ZodType<TraceScoreAverage> = z
|
|
50
|
+
.object({
|
|
51
|
+
average: z.number().describe(
|
|
52
|
+
"Mean of those scores. A binary evaluator averages 0 and 1, so its mean is the share that passed",
|
|
53
|
+
),
|
|
54
|
+
evaluator_uuid: z.string().describe(
|
|
55
|
+
"ID of the evaluator these numbers are for",
|
|
56
|
+
),
|
|
57
|
+
name: z.string().describe(
|
|
58
|
+
"Name of the evaluator. Uses the last stored name if it has been deleted",
|
|
59
|
+
),
|
|
60
|
+
output_type: TraceScoreAverageOutputType$zodSchema.describe(
|
|
61
|
+
"How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
|
|
62
|
+
),
|
|
63
|
+
scale_max: z.number().nullable().optional().describe(
|
|
64
|
+
"Highest value on a rating scale",
|
|
65
|
+
),
|
|
66
|
+
scale_min: z.number().nullable().optional().describe(
|
|
67
|
+
"Lowest value on a rating scale",
|
|
68
|
+
),
|
|
69
|
+
traces_scored: z.int().describe(
|
|
70
|
+
"How many of the matching traces this evaluator scored",
|
|
71
|
+
),
|
|
72
|
+
});
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod";
|
|
6
|
+
import { ClosedEnum } from "../types/enums.js";
|
|
7
|
+
|
|
8
|
+
export const TraceScoreResultEvaluatorType = {
|
|
9
|
+
Tts: "tts",
|
|
10
|
+
Stt: "stt",
|
|
11
|
+
Llm: "llm",
|
|
12
|
+
LlmGeneral: "llm-general",
|
|
13
|
+
Conversation: "conversation",
|
|
14
|
+
ToolCall: "tool-call",
|
|
15
|
+
} as const;
|
|
16
|
+
export type TraceScoreResultEvaluatorType = ClosedEnum<
|
|
17
|
+
typeof TraceScoreResultEvaluatorType
|
|
18
|
+
>;
|
|
19
|
+
|
|
20
|
+
export const TraceScoreResultEvaluatorType$zodSchema = z.enum([
|
|
21
|
+
"tts",
|
|
22
|
+
"stt",
|
|
23
|
+
"llm",
|
|
24
|
+
"llm-general",
|
|
25
|
+
"conversation",
|
|
26
|
+
"tool-call",
|
|
27
|
+
]);
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* How the evaluator scores:
|
|
31
|
+
*
|
|
32
|
+
* @remarks
|
|
33
|
+
*
|
|
34
|
+
* - `binary`: pass or fail
|
|
35
|
+
* - `rating`: a numeric score, using the scale in `output_config`
|
|
36
|
+
*/
|
|
37
|
+
export const TraceScoreResultOutputType = {
|
|
38
|
+
Binary: "binary",
|
|
39
|
+
Rating: "rating",
|
|
40
|
+
} as const;
|
|
41
|
+
/**
|
|
42
|
+
* How the evaluator scores:
|
|
43
|
+
*
|
|
44
|
+
* @remarks
|
|
45
|
+
*
|
|
46
|
+
* - `binary`: pass or fail
|
|
47
|
+
* - `rating`: a numeric score, using the scale in `output_config`
|
|
48
|
+
*/
|
|
49
|
+
export type TraceScoreResultOutputType = ClosedEnum<
|
|
50
|
+
typeof TraceScoreResultOutputType
|
|
51
|
+
>;
|
|
52
|
+
|
|
53
|
+
export const TraceScoreResultOutputType$zodSchema = z.enum([
|
|
54
|
+
"binary",
|
|
55
|
+
"rating",
|
|
56
|
+
]).describe(
|
|
57
|
+
"How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
export type TraceScoreResult = {
|
|
61
|
+
evaluator_uuid: string;
|
|
62
|
+
name: string;
|
|
63
|
+
evaluator_type?: TraceScoreResultEvaluatorType | null | undefined;
|
|
64
|
+
output_type: TraceScoreResultOutputType;
|
|
65
|
+
scale_min?: number | null | undefined;
|
|
66
|
+
scale_max?: number | null | undefined;
|
|
67
|
+
value: number;
|
|
68
|
+
reasoning?: string | null | undefined;
|
|
69
|
+
evaluator_version_id: string;
|
|
70
|
+
passed: boolean;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
export const TraceScoreResult$zodSchema: z.ZodType<TraceScoreResult> = z.object(
|
|
74
|
+
{
|
|
75
|
+
evaluator_type: TraceScoreResultEvaluatorType$zodSchema.nullable()
|
|
76
|
+
.optional().describe(
|
|
77
|
+
"What the evaluator judges:\n\n- `tts`: TTS audio\n- `stt`: one transcript\n- `llm`: a reply with its conversation history\n- `llm-general`: a standalone input and output pair\n- `conversation`: a full conversation\n- `tool-call`: whether the agent called the right tool, labelled by a person\n",
|
|
78
|
+
),
|
|
79
|
+
evaluator_uuid: z.string().describe(
|
|
80
|
+
"ID of the evaluator that produced this result",
|
|
81
|
+
),
|
|
82
|
+
evaluator_version_id: z.string().describe(
|
|
83
|
+
"ID of the evaluator version that produced this result",
|
|
84
|
+
),
|
|
85
|
+
name: z.string().describe(
|
|
86
|
+
"Name of the evaluator. Uses the last stored name if it has been deleted",
|
|
87
|
+
),
|
|
88
|
+
output_type: TraceScoreResultOutputType$zodSchema.describe(
|
|
89
|
+
"How the evaluator scores:\n\n- `binary`: pass or fail\n- `rating`: a numeric score, using the scale in `output_config`\n",
|
|
90
|
+
),
|
|
91
|
+
passed: z.boolean().describe(
|
|
92
|
+
"Whether this evaluator passed. A binary result passes on 1. A rating passes at the top of its scale",
|
|
93
|
+
),
|
|
94
|
+
reasoning: z.string().nullable().optional().describe(
|
|
95
|
+
"The judge's rationale for this result",
|
|
96
|
+
),
|
|
97
|
+
scale_max: z.number().nullable().optional().describe(
|
|
98
|
+
"Highest value on a rating scale",
|
|
99
|
+
),
|
|
100
|
+
scale_min: z.number().nullable().optional().describe(
|
|
101
|
+
"Lowest value on a rating scale",
|
|
102
|
+
),
|
|
103
|
+
value: z.number().describe(
|
|
104
|
+
"The judged result. Binary stores 0 or 1. Rating stores the numeric score",
|
|
105
|
+
),
|
|
106
|
+
},
|
|
107
|
+
);
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod";
|
|
6
|
+
import {
|
|
7
|
+
TraceEvalRunStatus,
|
|
8
|
+
TraceEvalRunStatus$zodSchema,
|
|
9
|
+
} from "./traceevalrunstatus.js";
|
|
10
|
+
import {
|
|
11
|
+
TraceMetadataEntry,
|
|
12
|
+
TraceMetadataEntry$zodSchema,
|
|
13
|
+
} from "./tracemetadataentry.js";
|
|
14
|
+
import { TraceOutput, TraceOutput$zodSchema } from "./traceoutput.js";
|
|
15
|
+
import {
|
|
16
|
+
TraceScoreResult,
|
|
17
|
+
TraceScoreResult$zodSchema,
|
|
18
|
+
} from "./tracescoreresult.js";
|
|
19
|
+
import { TraceToolCall, TraceToolCall$zodSchema } from "./tracetoolcall.js";
|
|
20
|
+
import { TraceTurn, TraceTurn$zodSchema } from "./traceturn.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format. Sent only with `include_content`
|
|
24
|
+
*/
|
|
25
|
+
export type TraceSummaryInput = string | Array<TraceTurn>;
|
|
26
|
+
|
|
27
|
+
export const TraceSummaryInput$zodSchema: z.ZodType<TraceSummaryInput> = z
|
|
28
|
+
.union([
|
|
29
|
+
z.string(),
|
|
30
|
+
z.array(TraceTurn$zodSchema),
|
|
31
|
+
]).describe(
|
|
32
|
+
"What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format. Sent only with `include_content`",
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
export type TraceSummary = {
|
|
36
|
+
uuid: string;
|
|
37
|
+
agent_id: string;
|
|
38
|
+
message_id?: string | null | undefined;
|
|
39
|
+
conversation_id?: string | null | undefined;
|
|
40
|
+
input_preview?: string | null | undefined;
|
|
41
|
+
response_preview?: string | null | undefined;
|
|
42
|
+
tool_names: Array<string>;
|
|
43
|
+
tool_calls: Array<TraceToolCall>;
|
|
44
|
+
turn_count: number;
|
|
45
|
+
tool_call_count: number;
|
|
46
|
+
metadata_count: number;
|
|
47
|
+
labels: Array<string>;
|
|
48
|
+
input?: string | Array<TraceTurn> | null | undefined;
|
|
49
|
+
output?: TraceOutput | null | undefined;
|
|
50
|
+
metadata?: Array<TraceMetadataEntry> | null | undefined;
|
|
51
|
+
created_at: string;
|
|
52
|
+
latest_run_status?: TraceEvalRunStatus | null | undefined;
|
|
53
|
+
latest_run_error?: string | null | undefined;
|
|
54
|
+
results?: Array<TraceScoreResult> | undefined;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export const TraceSummary$zodSchema: z.ZodType<TraceSummary> = z.object({
|
|
58
|
+
agent_id: z.string().describe("ID of the agent that produced the turn"),
|
|
59
|
+
conversation_id: z.string().nullable().optional().describe(
|
|
60
|
+
"The conversation ID you sent, if any",
|
|
61
|
+
),
|
|
62
|
+
created_at: z.string().describe("When the trace was created (ISO 8601 UTC)"),
|
|
63
|
+
input: z.union([
|
|
64
|
+
z.string(),
|
|
65
|
+
z.array(TraceTurn$zodSchema),
|
|
66
|
+
]).nullable().optional().describe(
|
|
67
|
+
"What the agent was given for this turn. For a `general` agent, the standalone prompt as a string. For a `conversation` agent, the history up to the reported output, oldest turn first, in OpenAI chat format. Sent only with `include_content`",
|
|
68
|
+
),
|
|
69
|
+
input_preview: z.string().nullable().optional().describe(
|
|
70
|
+
"The standalone prompt or the last user message, truncated for display",
|
|
71
|
+
),
|
|
72
|
+
labels: z.array(z.string()).describe(
|
|
73
|
+
"Your own tags for this turn, such as an environment or a release. Matched exactly when filtering, so keep the spelling stable",
|
|
74
|
+
),
|
|
75
|
+
latest_run_error: z.string().nullable().optional().describe(
|
|
76
|
+
"Why the latest scoring run was skipped or failed",
|
|
77
|
+
),
|
|
78
|
+
latest_run_status: TraceEvalRunStatus$zodSchema.nullable().optional()
|
|
79
|
+
.describe("Status of the latest scoring run for this trace"),
|
|
80
|
+
message_id: z.string().nullable().optional().describe(
|
|
81
|
+
"The message ID you sent, if any",
|
|
82
|
+
),
|
|
83
|
+
metadata: z.array(TraceMetadataEntry$zodSchema).nullable().optional()
|
|
84
|
+
.describe(
|
|
85
|
+
"Key-value pairs stored with the trace. Sent only with `include_content`",
|
|
86
|
+
),
|
|
87
|
+
metadata_count: z.int().describe(
|
|
88
|
+
"Number of metadata entries stored with the trace",
|
|
89
|
+
),
|
|
90
|
+
output: TraceOutput$zodSchema.nullable().optional().describe(
|
|
91
|
+
"What the agent produced for this turn. Sent only with `include_content`",
|
|
92
|
+
),
|
|
93
|
+
response_preview: z.string().nullable().optional().describe(
|
|
94
|
+
"The agent reply, truncated for display",
|
|
95
|
+
),
|
|
96
|
+
results: z.array(TraceScoreResult$zodSchema).optional().describe(
|
|
97
|
+
"Results for each evaluator on the latest scoring run",
|
|
98
|
+
),
|
|
99
|
+
tool_call_count: z.int().describe(
|
|
100
|
+
"Number of tool calls the agent issued for this turn",
|
|
101
|
+
),
|
|
102
|
+
tool_calls: z.array(TraceToolCall$zodSchema).describe(
|
|
103
|
+
"Tools the agent issued on this turn, with the arguments it passed",
|
|
104
|
+
),
|
|
105
|
+
tool_names: z.array(z.string()).describe(
|
|
106
|
+
"Names of the tools the agent issued on this turn, in order",
|
|
107
|
+
),
|
|
108
|
+
turn_count: z.int().describe(
|
|
109
|
+
"Number of turns in the stored input. A standalone prompt counts as 1",
|
|
110
|
+
),
|
|
111
|
+
uuid: z.string().describe("Unique ID for the trace"),
|
|
112
|
+
});
|
package/src/tool-names.ts
CHANGED
|
@@ -183,5 +183,9 @@ export const toolNames: Array<{ name: string; description: string }>= [
|
|
|
183
183
|
{
|
|
184
184
|
"name": "create-trace",
|
|
185
185
|
"description": "Store one production agent turn against an agent in the caller's org:\nthe conversation history as input, what the agent produced as output,\nand optional metadata. Use this to capture live traffic for later\nreview and conversion into tests. Fails with 429 once the workspace\nhits its trace limit.\n"
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
"name": "list-traces",
|
|
189
|
+
"description": "List the traces in the caller's org, newest first, one page at a\ntime. Each trace carries a short preview of its last user message\nand reply, its tool calls and labels, and the scores from its\nlatest scoring run with each evaluator's reasoning and evaluator\nversion. Filter by agent, label, creation time, text, or score.\nSet include_content to also get each trace's full input, output\nand metadata.\n"
|
|
186
190
|
}
|
|
187
191
|
];
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"workspaceCurrent.d.ts","sourceRoot":"","sources":["../../src/funcs/workspaceCurrent.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAI9C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EACL,iDAAiD,EAElD,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAW,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,gBAAgB,EACzB,OAAO,CAAC,EAAE,iDAAiD,GAAG,SAAS,EACvE,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CACX,MAAM,CACJ,QAAQ,EACN,QAAQ,GACR,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,mBAAmB,GACnB,mBAAmB,GACnB,eAAe,CAClB,CACF,CAMA"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"workspaceCurrent.js","sourceRoot":"","sources":["../../src/funcs/workspaceCurrent.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAU3C,OAAO,EAEL,2DAA2D,GAC5D,MAAM,2DAA2D,CAAC;AACnE,OAAO,EAAW,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGxD;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAyB,EACzB,OAAuE,EACvE,OAAwB;IAaxB,OAAO,IAAI,UAAU,CAAC,GAAG,CACvB,OAAO,EACP,OAAO,EACP,OAAO,CACR,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,GAAG,CAChB,OAAyB,EACzB,OAAuE,EACvE,OAAwB;IAgBxB,MAAM,OAAO,GAAG,SAAS,CACvB,OAAO,EACP,CAAC,MAAM,EAAE,EAAE,CACT,2DAA2D,CAAC,QAAQ,EAAE;SACnE,KAAK,CAAC,MAAM,CAAC,EAClB,yBAAyB,CAC1B,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC;IACnB,MAAM,KAAK,GAAG,UAAU,CAAC,wBAAwB,CAAC,EAAE,CAAC;IAErD,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAC,UAAU,CAAC;QACtC,MAAM,EAAE,kBAAkB;QAC1B,WAAW,EAAE,YAAY,CAAC,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE;YACxD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,MAAM;SACrB,CAAC;KACH,CAAC,CAAC,CAAC;IACJ,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvE,MAAM,eAAe,GAAG,qBAAqB,CAAC,aAAa,CAAC,CAAC;IAE7D,MAAM,OAAO,GAAG;QACd,OAAO,EAAE,OAAO,CAAC,QAAQ;QACzB,OAAO,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,QAAQ,IAAI,EAAE;QACrD,WAAW,EAAE,iDAAiD;QAC9D,YAAY,EAAE,IAAI;QAClB,gBAAgB,EAAE,eAAe;QACjC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ;QACzC,WAAW,EAAE,OAAO,EAAE,OAAO;eACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;eAC5B,EAAE,QAAQ,EAAE,MAAM,EAAE;QACzB,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI;YACjC,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;YACL,KAAK;SACN;KACF,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE;QACjD,QAAQ,EAAE,eAAe;QACzB,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,OAAO,EAAE,SAAS;QAC3B,IAAI,EAAE,KAAK;QACX,OAAO,EAAE,QAAQ;QACjB,IAAI,EAAE,KAAK;QACX,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;QACrC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS;eACtD,CAAC,CAAC;KACR,EAAE,OAAO,CAAC,CAAC;IACZ,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC;IAE9B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;QACvC,OAAO;QACP,UAAU,EAAE,EAAE;QACd,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE;YAChB,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,QAAQ,CAAC,KAAK;SACzB,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"workspaceCurrent.d.ts","sourceRoot":"","sources":["../../../src/mcp-server/tools/workspaceCurrent.ts"],"names":[],"mappings":"AAMA,OAAO,EAAgB,cAAc,EAAE,MAAM,aAAa,CAAC;AAE3D,QAAA,MAAM,IAAI;;CAGT,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,cAAc,CAAC,OAAO,IAAI,CAiC7D,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"workspaceCurrent.js","sourceRoot":"","sources":["../../../src/mcp-server/tools/workspaceCurrent.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,2DAA2D,EAAE,MAAM,8DAA8D,CAAC;AAC3I,OAAO,EAAE,YAAY,EAAkB,MAAM,aAAa,CAAC;AAE3D,MAAM,IAAI,GAAG;IACX,OAAO,EAAE,2DAA2D;SACjE,QAAQ,EAAE;CACd,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAgC;IAChE,IAAI,EAAE,uBAAuB;IAC7B,WAAW,EAAE;;;;;CAKd;IACC,MAAM,EAAE,CAAC,MAAM,CAAC;IAChB,WAAW,EAAE;QACX,OAAO,EAAE,2BAA2B;QACpC,iBAAiB,EAAE,KAAK;QACxB,gBAAgB,EAAE,IAAI;QACtB,eAAe,EAAE,IAAI;QACrB,cAAc,EAAE,IAAI;KACrB;IACD,IAAI;IACJ,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;QAChC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,gBAAgB,CACrC,MAAM,EACN,IAAI,CAAC,OAAO,EACZ,EAAE,YAAY,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CACzC,CAAC,QAAQ,EAAE,CAAC;QAEb,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACvD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,OAAO,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;CACF,CAAC"}
|