@vertesia/common 1.5.0-dev.20260910.052912Z → 1.6.0-dev.20260912.110338Z
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/access-control-values.d.ts +1 -0
- package/lib/access-control-values.d.ts.map +1 -1
- package/lib/access-control-values.js +1 -0
- package/lib/access-control-values.js.map +1 -1
- package/lib/api-contract/components.generated.json +4188 -2267
- package/lib/api-schemas/adapter.d.ts.map +1 -1
- package/lib/api-schemas/adapter.js +8 -1
- package/lib/api-schemas/adapter.js.map +1 -1
- package/lib/api-schemas/agent-communication.d.ts +1 -0
- package/lib/api-schemas/agent-communication.d.ts.map +1 -1
- package/lib/api-schemas/agent-communication.js +6 -0
- package/lib/api-schemas/agent-communication.js.map +1 -1
- package/lib/api-schemas/agent-runs.d.ts +5271 -903
- package/lib/api-schemas/agent-runs.d.ts.map +1 -1
- package/lib/api-schemas/agent-runs.js +343 -1
- package/lib/api-schemas/agent-runs.js.map +1 -1
- package/lib/api-schemas/app-runtime.d.ts +1240 -22
- package/lib/api-schemas/app-runtime.d.ts.map +1 -1
- package/lib/api-schemas/content.d.ts +435 -0
- package/lib/api-schemas/content.d.ts.map +1 -1
- package/lib/api-schemas/data-store.d.ts +1 -0
- package/lib/api-schemas/data-store.d.ts.map +1 -1
- package/lib/api-schemas/data-store.js +7 -0
- package/lib/api-schemas/data-store.js.map +1 -1
- package/lib/api-schemas/delegation.d.ts +35 -0
- package/lib/api-schemas/delegation.d.ts.map +1 -0
- package/lib/api-schemas/delegation.js +21 -0
- package/lib/api-schemas/delegation.js.map +1 -0
- package/lib/api-schemas/document-processing.d.ts +261 -0
- package/lib/api-schemas/document-processing.d.ts.map +1 -1
- package/lib/api-schemas/emit-json-schema.d.ts +5 -0
- package/lib/api-schemas/emit-json-schema.d.ts.map +1 -0
- package/lib/api-schemas/emit-json-schema.js +37 -0
- package/lib/api-schemas/emit-json-schema.js.map +1 -0
- package/lib/api-schemas/environment.d.ts +18 -0
- package/lib/api-schemas/environment.d.ts.map +1 -1
- package/lib/api-schemas/events.d.ts +783 -0
- package/lib/api-schemas/events.d.ts.map +1 -1
- package/lib/api-schemas/index.d.ts +1 -0
- package/lib/api-schemas/index.d.ts.map +1 -1
- package/lib/api-schemas/index.js +1 -0
- package/lib/api-schemas/index.js.map +1 -1
- package/lib/api-schemas/interaction.d.ts +3939 -91
- package/lib/api-schemas/interaction.d.ts.map +1 -1
- package/lib/api-schemas/interaction.js +14 -3
- package/lib/api-schemas/interaction.js.map +1 -1
- package/lib/api-schemas/oauth-server.d.ts +1 -0
- package/lib/api-schemas/oauth-server.d.ts.map +1 -1
- package/lib/api-schemas/oauth-server.js +3 -0
- package/lib/api-schemas/oauth-server.js.map +1 -1
- package/lib/api-schemas/process-agent-policy.d.ts +69 -0
- package/lib/api-schemas/process-agent-policy.d.ts.map +1 -0
- package/lib/api-schemas/process-agent-policy.js +154 -0
- package/lib/api-schemas/process-agent-policy.js.map +1 -0
- package/lib/api-schemas/process.d.ts +1260 -0
- package/lib/api-schemas/process.d.ts.map +1 -1
- package/lib/api-schemas/process.js +29 -1
- package/lib/api-schemas/process.js.map +1 -1
- package/lib/api-schemas/project-configuration.d.ts +878 -8
- package/lib/api-schemas/project-configuration.d.ts.map +1 -1
- package/lib/api-schemas/project.d.ts +1327 -22
- package/lib/api-schemas/project.d.ts.map +1 -1
- package/lib/api-schemas/registry.d.ts +21836 -3256
- package/lib/api-schemas/registry.d.ts.map +1 -1
- package/lib/api-schemas/registry.js +40 -7
- package/lib/api-schemas/registry.js.map +1 -1
- package/lib/api-schemas/store.d.ts +4427 -77
- package/lib/api-schemas/store.d.ts.map +1 -1
- package/lib/api-schemas/sts.d.ts +6 -0
- package/lib/api-schemas/sts.d.ts.map +1 -1
- package/lib/api-schemas/sts.js +3 -0
- package/lib/api-schemas/sts.js.map +1 -1
- package/lib/api-schemas/workflow-runs.d.ts +189 -0
- package/lib/api-schemas/workflow-runs.d.ts.map +1 -1
- package/lib/api-schemas/workflow-runs.js +14 -0
- package/lib/api-schemas/workflow-runs.js.map +1 -1
- package/lib/apikey.d.ts +1 -0
- package/lib/apikey.d.ts.map +1 -1
- package/lib/apikey.js.map +1 -1
- package/lib/delegation.d.ts +13 -0
- package/lib/delegation.d.ts.map +1 -0
- package/lib/delegation.js +2 -0
- package/lib/delegation.js.map +1 -0
- package/lib/environment.d.ts +1 -0
- package/lib/environment.d.ts.map +1 -1
- package/lib/index.d.ts +1 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/oauth-scopes.d.ts.map +1 -1
- package/lib/oauth-scopes.js +1 -0
- package/lib/oauth-scopes.js.map +1 -1
- package/lib/store/agent-run-values.d.ts +25 -0
- package/lib/store/agent-run-values.d.ts.map +1 -0
- package/lib/store/agent-run-values.js +25 -0
- package/lib/store/agent-run-values.js.map +1 -0
- package/lib/store/agent-run.d.ts +22 -2
- package/lib/store/agent-run.d.ts.map +1 -1
- package/lib/store/agent-run.js +1 -1
- package/lib/store/agent-run.js.map +1 -1
- package/lib/store/conversation-state.d.ts +1 -1
- package/lib/store/conversation-state.d.ts.map +1 -1
- package/lib/store/intake-policy-schema.generated.d.ts.map +1 -1
- package/lib/store/intake-policy-schema.generated.js +124 -0
- package/lib/store/intake-policy-schema.generated.js.map +1 -1
- package/lib/store/process.d.ts +17 -0
- package/lib/store/process.d.ts.map +1 -1
- package/lib/store/process.js.map +1 -1
- package/lib/store/schedule.d.ts +62 -25
- package/lib/store/schedule.d.ts.map +1 -1
- package/lib/store/schedule.js +2 -2
- package/lib/store/schedule.js.map +1 -1
- package/lib/store/workflow.d.ts +2 -0
- package/lib/store/workflow.d.ts.map +1 -1
- package/lib/store/workflow.js.map +1 -1
- package/lib/vertesia-common.js +2 -2
- package/lib/vertesia-common.js.map +1 -1
- package/lib/workflow-analytics.d.ts +198 -2
- package/lib/workflow-analytics.d.ts.map +1 -1
- package/lib/workflow-analytics.js +6 -0
- package/lib/workflow-analytics.js.map +1 -1
- package/package.json +8 -8
- package/src/access-control-values.ts +1 -0
- package/src/api-contract/components.generated.json +4188 -2267
- package/src/api-schemas/adapter-edge-cases.test.ts +19 -0
- package/src/api-schemas/adapter.ts +8 -1
- package/src/api-schemas/agent-communication.ts +6 -0
- package/src/api-schemas/agent-history.contract.test.ts +42 -0
- package/src/api-schemas/agent-runs.contract.test.ts +203 -0
- package/src/api-schemas/agent-runs.ts +374 -0
- package/src/api-schemas/data-store.contract.test.ts +6 -0
- package/src/api-schemas/data-store.ts +8 -0
- package/src/api-schemas/delegation.ts +21 -0
- package/src/api-schemas/emit-json-schema.test.ts +36 -0
- package/src/api-schemas/emit-json-schema.ts +40 -0
- package/src/api-schemas/environment.contract.test.ts +3 -0
- package/src/api-schemas/index.ts +1 -0
- package/src/api-schemas/interaction.contract.test.ts +23 -0
- package/src/api-schemas/interaction.ts +15 -3
- package/src/api-schemas/llumiverse.contract.test.ts +3 -9
- package/src/api-schemas/oauth-server.ts +4 -0
- package/src/api-schemas/process-agent-policy.ts +167 -0
- package/src/api-schemas/process.contract.test.ts +115 -0
- package/src/api-schemas/process.ts +31 -1
- package/src/api-schemas/registry.ts +43 -10
- package/src/api-schemas/sts.ts +3 -0
- package/src/api-schemas/workflow-runs.ts +16 -0
- package/src/api-schemas/zeno-response-contracts.test.ts +36 -0
- package/src/apikey.ts +1 -0
- package/src/delegation.ts +18 -0
- package/src/index.ts +6 -0
- package/src/oauth-scopes.ts +1 -0
- package/src/store/agent-run-values.ts +27 -0
- package/src/store/agent-run.ts +36 -0
- package/src/store/conversation-state.ts +1 -1
- package/src/store/intake-policy-schema.generated.ts +125 -0
- package/src/store/process.ts +22 -0
- package/src/store/schedule.test.ts +59 -0
- package/src/store/schedule.ts +91 -27
- package/src/store/workflow.ts +2 -0
- package/src/workflow-analytics.ts +238 -1
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import { ExecutionTokenUsageSchema, ReasoningEffortSchema } from '@llumiverse/common/schemas';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
+
import { AGENT_RUN_FEEDBACK_COMMENT_MAX_LENGTH, AGENT_RUN_FEEDBACK_ID_MAX_LENGTH } from '../store/agent-run-values.js';
|
|
5
6
|
import type { AgentMessageType, FileProcessingStatus } from '../store/workflow.js';
|
|
6
7
|
import { type AgentEvent, AgentEventType, LlmCallType, TelemetryToolType } from '../workflow-analytics.js';
|
|
7
8
|
import * as AppLifecycleSchemas from './app-lifecycle.js';
|
|
8
9
|
import {
|
|
9
10
|
AgentRunStatusSchema,
|
|
11
|
+
AgentToolApprovalClassSchema,
|
|
10
12
|
ContentObjectTypeRefSchema,
|
|
11
13
|
ConversationActivityStateSchema,
|
|
12
14
|
EventRefSchema,
|
|
@@ -36,6 +38,244 @@ import { AgentCheckpointConfigurationSchema } from './project-configuration.js';
|
|
|
36
38
|
import { nullableStringSchema } from './schema-primitives.js';
|
|
37
39
|
import { InteractionExecutionConfigurationSchema } from './store.js';
|
|
38
40
|
|
|
41
|
+
// ----------------------------------------------------------------------------
|
|
42
|
+
// Evaluation vocabularies
|
|
43
|
+
// ----------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
export const TurnTerminalTypeSchema = z
|
|
46
|
+
.enum(['answer', 'user_stopped', 'completed', 'failed', 'cancelled', 'interrupted'])
|
|
47
|
+
.meta({ id: 'TurnTerminalType', description: 'How an agent turn ended.' });
|
|
48
|
+
|
|
49
|
+
export const EvaluationSeveritySchema = z.enum(['none', 'low', 'medium', 'high']).meta({
|
|
50
|
+
id: 'EvaluationSeverity',
|
|
51
|
+
description: 'Worst deterministic detector level. `none` means no detector fired, not success.',
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
export const TurnEvaluationFlagSchema = z
|
|
55
|
+
.enum([
|
|
56
|
+
'user_stopped_after_failure',
|
|
57
|
+
'mutation_unsuccessful',
|
|
58
|
+
'run_failed',
|
|
59
|
+
'unrecovered_tool',
|
|
60
|
+
'fail_streak',
|
|
61
|
+
'identical_retry',
|
|
62
|
+
'reread',
|
|
63
|
+
'high_gather',
|
|
64
|
+
'overhead',
|
|
65
|
+
'followup_after_answer',
|
|
66
|
+
'approval_denied',
|
|
67
|
+
])
|
|
68
|
+
.meta({ id: 'TurnEvaluationFlag', description: 'Reason behind an evaluation severity.' });
|
|
69
|
+
|
|
70
|
+
export const ToolErrorClassSchema = z
|
|
71
|
+
.enum(['schema', 'platform', 'config', 'environment', 'other'])
|
|
72
|
+
.meta({ id: 'ToolErrorClass', description: 'Coarse class of a tool error, derived from the error text.' });
|
|
73
|
+
|
|
74
|
+
const ToolErrorClassCountsSchema = z.strictObject({
|
|
75
|
+
schema: z.number().int().min(0),
|
|
76
|
+
platform: z.number().int().min(0),
|
|
77
|
+
config: z.number().int().min(0),
|
|
78
|
+
environment: z.number().int().min(0),
|
|
79
|
+
other: z.number().int().min(0),
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
const TelemetryDeploymentSchema = z.strictObject({
|
|
83
|
+
env: z.string(),
|
|
84
|
+
group: z.string(),
|
|
85
|
+
version: z.string(),
|
|
86
|
+
region: z.string().optional(),
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
const TelemetryProducerSchema = z.strictObject({
|
|
90
|
+
version: z.string(),
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
export const JudgeGateReasonSchema = z
|
|
94
|
+
.enum(['signal', 'sample'])
|
|
95
|
+
.meta({ id: 'JudgeGateReason', description: 'Why the judge looked at a run.' });
|
|
96
|
+
|
|
97
|
+
export const JudgeOutcomeSchema = z
|
|
98
|
+
.enum(['judged', 'skipped_unarchived', 'failed'])
|
|
99
|
+
.meta({ id: 'JudgeOutcome', description: 'What a judge run produced.' });
|
|
100
|
+
|
|
101
|
+
export const JudgeVerdictSchema = z
|
|
102
|
+
.enum(['success', 'partial', 'failure'])
|
|
103
|
+
.meta({ id: 'JudgeVerdict', description: "The judge's reading of a turn." });
|
|
104
|
+
|
|
105
|
+
// ----------------------------------------------------------------------------
|
|
106
|
+
// User feedback
|
|
107
|
+
// ----------------------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* How the user rated an agent run's work. Two values on purpose: a rating is a signal, not a score,
|
|
111
|
+
* and a wider scale invites a precision the reader does not have.
|
|
112
|
+
*/
|
|
113
|
+
export const AgentRunFeedbackRatingSchema = z
|
|
114
|
+
.enum(['up', 'down'])
|
|
115
|
+
.meta({ id: 'AgentRunFeedbackRating', description: 'Thumbs up or thumbs down on an agent run.' });
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The closed reason vocabulary. Closed rather than free text because the reason is the part that
|
|
119
|
+
* gets counted; the free-text `comment` stays with the tenant and is never aggregated.
|
|
120
|
+
*/
|
|
121
|
+
export const AgentRunFeedbackReasonCodeSchema = z
|
|
122
|
+
.enum([
|
|
123
|
+
'accurate',
|
|
124
|
+
'helpful',
|
|
125
|
+
'fast',
|
|
126
|
+
'well_explained',
|
|
127
|
+
'wrong_result',
|
|
128
|
+
'incomplete',
|
|
129
|
+
'misunderstood_request',
|
|
130
|
+
'too_slow',
|
|
131
|
+
'tool_failure',
|
|
132
|
+
'unsafe_action',
|
|
133
|
+
'other',
|
|
134
|
+
])
|
|
135
|
+
.meta({ id: 'AgentRunFeedbackReasonCode', description: 'Why the run was rated the way it was.' });
|
|
136
|
+
|
|
137
|
+
export const AgentRunFeedbackPayloadSchema = z
|
|
138
|
+
.strictObject({
|
|
139
|
+
feedback_id: z.string().min(1).max(AGENT_RUN_FEEDBACK_ID_MAX_LENGTH).meta({
|
|
140
|
+
description: 'Client-generated idempotency key (a UUID). A retried request with the same id is a no-op.',
|
|
141
|
+
}),
|
|
142
|
+
rating: AgentRunFeedbackRatingSchema,
|
|
143
|
+
reason_code: AgentRunFeedbackReasonCodeSchema.optional(),
|
|
144
|
+
comment: z.string().max(AGENT_RUN_FEEDBACK_COMMENT_MAX_LENGTH).optional().meta({
|
|
145
|
+
description: 'Free-text comment. Stored with the tenant; only its presence is ever counted.',
|
|
146
|
+
}),
|
|
147
|
+
message_id: z.string().optional().meta({
|
|
148
|
+
description: 'Rate one message rather than the run as a whole.',
|
|
149
|
+
}),
|
|
150
|
+
message_seq: z.number().int().min(0).optional().meta({
|
|
151
|
+
description: 'Position of the rated message in the run, when a message is rated.',
|
|
152
|
+
}),
|
|
153
|
+
})
|
|
154
|
+
.meta({ id: 'AgentRunFeedbackPayload', description: 'A user rating on an agent run.' });
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* What happened to the rating. `recorded` and `replaced` both mean it counts; `replaced` says an
|
|
158
|
+
* earlier rating by the same user on the same scope was superseded. `disabled` means this
|
|
159
|
+
* deployment does not record feedback; it answers 200 because a rating is never the user's error.
|
|
160
|
+
*/
|
|
161
|
+
export const AgentRunFeedbackStatusSchema = z
|
|
162
|
+
.enum(['recorded', 'replaced', 'disabled'])
|
|
163
|
+
.meta({ id: 'AgentRunFeedbackStatus', description: 'Whether the rating was recorded.' });
|
|
164
|
+
|
|
165
|
+
export const AgentRunFeedbackCountsSchema = z
|
|
166
|
+
.strictObject({
|
|
167
|
+
up: z.number().int().min(0),
|
|
168
|
+
down: z.number().int().min(0),
|
|
169
|
+
last_rating: AgentRunFeedbackRatingSchema.optional(),
|
|
170
|
+
last_reason_code: AgentRunFeedbackReasonCodeSchema.optional(),
|
|
171
|
+
})
|
|
172
|
+
.meta({ id: 'AgentRunFeedbackCounts', description: 'Ratings over the retained feedback entries.' });
|
|
173
|
+
|
|
174
|
+
export const AgentRunFeedbackResponseSchema = z
|
|
175
|
+
.strictObject({
|
|
176
|
+
status: AgentRunFeedbackStatusSchema,
|
|
177
|
+
feedback_counts: AgentRunFeedbackCountsSchema.optional(),
|
|
178
|
+
})
|
|
179
|
+
.meta({ id: 'AgentRunFeedbackResponse', description: 'Result of rating an agent run.' });
|
|
180
|
+
|
|
181
|
+
export const AgentRunFeedbackEntrySchema = z
|
|
182
|
+
.strictObject({
|
|
183
|
+
feedback_id: z.string(),
|
|
184
|
+
rating: AgentRunFeedbackRatingSchema,
|
|
185
|
+
reason_code: AgentRunFeedbackReasonCodeSchema.optional(),
|
|
186
|
+
comment: z.string().optional(),
|
|
187
|
+
message_id: z.string().optional(),
|
|
188
|
+
message_seq: z.number().int().optional(),
|
|
189
|
+
message_scoped: z.boolean(),
|
|
190
|
+
user_id: z.string(),
|
|
191
|
+
rated_at: z.string().meta({ format: 'date-time' }),
|
|
192
|
+
replaced_at: z.string().meta({ format: 'date-time' }).optional(),
|
|
193
|
+
})
|
|
194
|
+
.meta({ id: 'AgentRunFeedbackEntry', description: 'One recorded rating on an agent run.' });
|
|
195
|
+
|
|
196
|
+
// ----------------------------------------------------------------------------
|
|
197
|
+
// Evaluation summary
|
|
198
|
+
// ----------------------------------------------------------------------------
|
|
199
|
+
|
|
200
|
+
const AgentRunEvaluationTotalsSchema = z.strictObject({
|
|
201
|
+
tool_calls: z.number().int(),
|
|
202
|
+
error_tool_results: z.number().int(),
|
|
203
|
+
unrecovered_tools: z.array(z.string()),
|
|
204
|
+
llm_calls: z.number().int(),
|
|
205
|
+
prompt_tokens: z.number(),
|
|
206
|
+
completion_tokens: z.number(),
|
|
207
|
+
cached_tokens: z.number(),
|
|
208
|
+
retry_completion_tokens: z.number(),
|
|
209
|
+
active_ms: z.number(),
|
|
210
|
+
ask_user_wait_ms: z.number(),
|
|
211
|
+
approval_wait_ms: z.number(),
|
|
212
|
+
followups: z.number().int(),
|
|
213
|
+
approvals_requested: z.number().int(),
|
|
214
|
+
approvals_denied: z.number().int(),
|
|
215
|
+
stop_requests: z.number().int(),
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* What the workflow reports about a run: the fold of its turn evaluations. Owned by the workflow;
|
|
220
|
+
* it never carries feedback or judge fields, which the server owns.
|
|
221
|
+
*/
|
|
222
|
+
export const AgentRunEvaluationRollupSchema = z
|
|
223
|
+
.strictObject({
|
|
224
|
+
seq: z.number().int().min(1).meta({
|
|
225
|
+
description: 'Monotonic per run; the server ignores a rollup older than the one it holds.',
|
|
226
|
+
}),
|
|
227
|
+
detector_version: z.number().int(),
|
|
228
|
+
turns: z.number().int(),
|
|
229
|
+
severity: EvaluationSeveritySchema,
|
|
230
|
+
flags: z.array(TurnEvaluationFlagSchema),
|
|
231
|
+
worst_turn_seq: z.number().int().optional(),
|
|
232
|
+
last_terminal_type: TurnTerminalTypeSchema.optional(),
|
|
233
|
+
terminal_error_class: z.string().optional(),
|
|
234
|
+
partial: z.boolean().optional(),
|
|
235
|
+
totals: AgentRunEvaluationTotalsSchema,
|
|
236
|
+
updated_at: z.string().meta({ format: 'date-time' }),
|
|
237
|
+
})
|
|
238
|
+
.meta({ id: 'AgentRunEvaluationRollup', description: 'Fold of the turn evaluations of a run.' });
|
|
239
|
+
|
|
240
|
+
export const AgentRunJudgeResultSchema = z
|
|
241
|
+
.strictObject({
|
|
242
|
+
rev: z.number().int(),
|
|
243
|
+
gate: JudgeGateReasonSchema,
|
|
244
|
+
sample_rate: z.number(),
|
|
245
|
+
selected_probability: z.number(),
|
|
246
|
+
outcome: JudgeOutcomeSchema,
|
|
247
|
+
verdict: JudgeVerdictSchema.optional(),
|
|
248
|
+
score: z.number().optional(),
|
|
249
|
+
model: z.string().optional(),
|
|
250
|
+
prompt_version: z.string(),
|
|
251
|
+
turns_judged: z.array(z.number().int()).optional(),
|
|
252
|
+
judged_at: z.string().meta({ format: 'date-time' }),
|
|
253
|
+
})
|
|
254
|
+
.meta({ id: 'AgentRunJudgeResult', description: 'Latest judge result for a run.' });
|
|
255
|
+
|
|
256
|
+
export const AgentRunContradictionReasonSchema = z
|
|
257
|
+
.enum(['feedback_down_on_clean_run', 'judge_failure_on_clean_run'])
|
|
258
|
+
.meta({ id: 'AgentRunContradictionReason' });
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* The server-owned evaluation summary of a run: the workflow rollup, the feedback counts and the
|
|
262
|
+
* judge result, plus the derived severity, flags and contradiction the run list filters on.
|
|
263
|
+
*/
|
|
264
|
+
export const AgentRunEvaluationSchema = z
|
|
265
|
+
.strictObject({
|
|
266
|
+
rev: z.number().int().meta({ description: 'Bumped on every change to any part of the summary.' }),
|
|
267
|
+
rollup: AgentRunEvaluationRollupSchema.optional(),
|
|
268
|
+
feedback_counts: AgentRunFeedbackCountsSchema.optional(),
|
|
269
|
+
judge: AgentRunJudgeResultSchema.optional(),
|
|
270
|
+
severity: EvaluationSeveritySchema,
|
|
271
|
+
flags: z.array(TurnEvaluationFlagSchema),
|
|
272
|
+
contradicted: z.boolean(),
|
|
273
|
+
contradiction_reasons: z.array(AgentRunContradictionReasonSchema).optional(),
|
|
274
|
+
deployment_env: z.string().optional(),
|
|
275
|
+
updated_at: z.string().meta({ format: 'date-time' }),
|
|
276
|
+
})
|
|
277
|
+
.meta({ id: 'AgentRunEvaluation', description: 'Evaluation summary of an agent run.' });
|
|
278
|
+
|
|
39
279
|
const agentEventBase = {
|
|
40
280
|
timestamp: z.string(),
|
|
41
281
|
runId: z.string(),
|
|
@@ -46,6 +286,9 @@ const agentEventBase = {
|
|
|
46
286
|
interactionId: z.string(),
|
|
47
287
|
parentRunId: z.string().optional(),
|
|
48
288
|
ancestorRunIds: z.array(z.string()).optional(),
|
|
289
|
+
eventId: z.string().optional(),
|
|
290
|
+
deployment: TelemetryDeploymentSchema.optional(),
|
|
291
|
+
producer: TelemetryProducerSchema.optional(),
|
|
49
292
|
};
|
|
50
293
|
|
|
51
294
|
const AgentRunStartedEventSchema = z.strictObject({
|
|
@@ -123,6 +366,96 @@ const ToolCallEventSchema = z.strictObject({
|
|
|
123
366
|
errorType: z.string().optional(),
|
|
124
367
|
errorMessage: z.string().optional(),
|
|
125
368
|
spawnedChildWorkflow: z.boolean().optional(),
|
|
369
|
+
approvalClass: AgentToolApprovalClassSchema.optional(),
|
|
370
|
+
errorClass: ToolErrorClassSchema.optional(),
|
|
371
|
+
signatureHash: z.string().optional(),
|
|
372
|
+
workstreamId: z.string().optional(),
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
const TurnToolObservationSchema = z.strictObject({
|
|
376
|
+
seq: z.number().int().min(1),
|
|
377
|
+
toolUseId: z.string(),
|
|
378
|
+
name: z.string(),
|
|
379
|
+
approvalClass: AgentToolApprovalClassSchema.optional(),
|
|
380
|
+
ok: z.boolean(),
|
|
381
|
+
errorClass: ToolErrorClassSchema.optional(),
|
|
382
|
+
sig: z.string(),
|
|
383
|
+
durationMs: z.number(),
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
const TurnEvaluationEventSchema = z.strictObject({
|
|
387
|
+
...agentEventBase,
|
|
388
|
+
eventType: z.literal(AgentEventType.TurnEvaluation),
|
|
389
|
+
schemaVersion: z.number().int(),
|
|
390
|
+
detectorVersion: z.number().int(),
|
|
391
|
+
workstreamId: z.string(),
|
|
392
|
+
turnSeq: z.number().int().min(1),
|
|
393
|
+
terminalType: TurnTerminalTypeSchema,
|
|
394
|
+
partial: z.boolean().optional(),
|
|
395
|
+
startedAt: z.string(),
|
|
396
|
+
endedAt: z.string(),
|
|
397
|
+
durationMs: z.number(),
|
|
398
|
+
activeMs: z.number(),
|
|
399
|
+
askUserWaitMs: z.number(),
|
|
400
|
+
approvalWaitMs: z.number(),
|
|
401
|
+
timeToFirstAnswerMs: z.number().optional(),
|
|
402
|
+
toolCalls: z.number().int(),
|
|
403
|
+
errorToolResults: z.number().int(),
|
|
404
|
+
maxFailStreak: z.number().int(),
|
|
405
|
+
identicalRetryCount: z.number().int(),
|
|
406
|
+
unrecoveredTools: z.array(z.string()),
|
|
407
|
+
errorClasses: ToolErrorClassCountsSchema,
|
|
408
|
+
gatherCalls: z.number().int(),
|
|
409
|
+
gatherRatio: z.number().optional(),
|
|
410
|
+
mutationAttempted: z.boolean(),
|
|
411
|
+
mutationSucceeded: z.boolean(),
|
|
412
|
+
rereadCount: z.number().int(),
|
|
413
|
+
overheadCalls: z.number().int(),
|
|
414
|
+
skillsLoaded: z.number().int(),
|
|
415
|
+
interruptedToolCalls: z.number().int(),
|
|
416
|
+
llmCalls: z.number().int(),
|
|
417
|
+
promptTokens: z.number(),
|
|
418
|
+
completionTokens: z.number(),
|
|
419
|
+
cachedTokens: z.number(),
|
|
420
|
+
retryCompletionTokens: z.number(),
|
|
421
|
+
approvalsRequested: z.number().int(),
|
|
422
|
+
approvalsDenied: z.number().int(),
|
|
423
|
+
stopRequests: z.number().int(),
|
|
424
|
+
followupAfterAnswer: z.boolean(),
|
|
425
|
+
severity: EvaluationSeveritySchema,
|
|
426
|
+
flags: z.array(TurnEvaluationFlagSchema),
|
|
427
|
+
terminalErrorClass: z.string().optional(),
|
|
428
|
+
tools: z.array(TurnToolObservationSchema),
|
|
429
|
+
toolsTruncated: z.number().int(),
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
const FeedbackEventSchema = z.strictObject({
|
|
433
|
+
...agentEventBase,
|
|
434
|
+
eventType: z.literal(AgentEventType.Feedback),
|
|
435
|
+
feedbackId: z.string(),
|
|
436
|
+
rating: AgentRunFeedbackRatingSchema,
|
|
437
|
+
reasonCode: AgentRunFeedbackReasonCodeSchema.optional(),
|
|
438
|
+
hasComment: z.boolean(),
|
|
439
|
+
messageScoped: z.boolean(),
|
|
440
|
+
messageSeq: z.number().int().optional(),
|
|
441
|
+
replaced: z.boolean(),
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
const TurnJudgementEventSchema = z.strictObject({
|
|
445
|
+
...agentEventBase,
|
|
446
|
+
eventType: z.literal(AgentEventType.TurnJudgement),
|
|
447
|
+
evaluationRev: z.number().int(),
|
|
448
|
+
workstreamId: z.string(),
|
|
449
|
+
turnSeq: z.number().int().min(0),
|
|
450
|
+
gate: JudgeGateReasonSchema,
|
|
451
|
+
sampleRate: z.number(),
|
|
452
|
+
selectedProbability: z.number(),
|
|
453
|
+
outcome: JudgeOutcomeSchema,
|
|
454
|
+
verdict: JudgeVerdictSchema.optional(),
|
|
455
|
+
score: z.number().optional(),
|
|
456
|
+
reasons: z.array(z.string()).optional(),
|
|
457
|
+
promptVersion: z.string(),
|
|
458
|
+
detectorVersion: z.number().int().optional(),
|
|
126
459
|
});
|
|
127
460
|
|
|
128
461
|
export const AgentEventSchema: z.ZodType<AgentEvent> = z
|
|
@@ -131,6 +464,9 @@ export const AgentEventSchema: z.ZodType<AgentEvent> = z
|
|
|
131
464
|
AgentRunCompletedEventSchema,
|
|
132
465
|
LlmCallEventSchema,
|
|
133
466
|
ToolCallEventSchema,
|
|
467
|
+
TurnEvaluationEventSchema,
|
|
468
|
+
FeedbackEventSchema,
|
|
469
|
+
TurnJudgementEventSchema,
|
|
134
470
|
])
|
|
135
471
|
.meta({ id: 'AgentEvent' });
|
|
136
472
|
|
|
@@ -420,6 +756,11 @@ export const AutonomousRunResponseSchema = z
|
|
|
420
756
|
.array(z.string())
|
|
421
757
|
.meta({ description: 'Lessons learned from the conversation (extracted at completion)' })
|
|
422
758
|
.optional(),
|
|
759
|
+
evaluation: AgentRunEvaluationSchema.meta({ description: 'Evaluation summary of the run.' }).optional(),
|
|
760
|
+
feedback: z
|
|
761
|
+
.array(AgentRunFeedbackEntrySchema)
|
|
762
|
+
.meta({ description: 'Retained user ratings on the run.' })
|
|
763
|
+
.optional(),
|
|
423
764
|
archived_at: z
|
|
424
765
|
.string()
|
|
425
766
|
.meta({ description: 'When the last successful archive completed', format: 'date-time' })
|
|
@@ -568,6 +909,11 @@ export const AgentRunSchema = z
|
|
|
568
909
|
.array(z.string())
|
|
569
910
|
.meta({ description: 'Lessons learned from the conversation (extracted at completion)' })
|
|
570
911
|
.optional(),
|
|
912
|
+
evaluation: AgentRunEvaluationSchema.meta({ description: 'Evaluation summary of the run.' }).optional(),
|
|
913
|
+
feedback: z
|
|
914
|
+
.array(AgentRunFeedbackEntrySchema)
|
|
915
|
+
.meta({ description: 'Retained user ratings on the run.' })
|
|
916
|
+
.optional(),
|
|
571
917
|
archived_at: z
|
|
572
918
|
.string()
|
|
573
919
|
.meta({ description: 'When the last successful archive completed', format: 'date-time' })
|
|
@@ -988,6 +1334,14 @@ export const AgentRunArtifactQuerySchema = z
|
|
|
988
1334
|
|
|
989
1335
|
export const AgentRunDetailsQuerySchema = z
|
|
990
1336
|
.object({
|
|
1337
|
+
from: z
|
|
1338
|
+
.string()
|
|
1339
|
+
.max(12000)
|
|
1340
|
+
.meta({
|
|
1341
|
+
description:
|
|
1342
|
+
'Opaque history cursor from next_from; requires include_history. Invalid or expired cursors return a snapshot.',
|
|
1343
|
+
})
|
|
1344
|
+
.optional(),
|
|
991
1345
|
include_history: z.boolean().optional(),
|
|
992
1346
|
hydrate_payloads: z.boolean().optional(),
|
|
993
1347
|
})
|
|
@@ -999,6 +1353,10 @@ export const AgentRunArtifactsQuerySchema = z
|
|
|
999
1353
|
})
|
|
1000
1354
|
.meta({ id: 'AgentRunArtifactsQuery' });
|
|
1001
1355
|
|
|
1356
|
+
export const ListAgentRunsEvaluationSeveritySchema = z
|
|
1357
|
+
.enum(['unrated', 'none', 'low', 'medium', 'high'])
|
|
1358
|
+
.meta({ id: 'ListAgentRunsEvaluationSeverity' });
|
|
1359
|
+
|
|
1002
1360
|
export const ListAgentRunsQuerySchema = z
|
|
1003
1361
|
.object({
|
|
1004
1362
|
id: z.string().meta({ description: 'Filter by agent run ID' }).optional(),
|
|
@@ -1022,6 +1380,21 @@ export const ListAgentRunsQuerySchema = z
|
|
|
1022
1380
|
run_kind: RunKindSchema.meta({ description: 'Filter by internal run discriminator' }).optional(),
|
|
1023
1381
|
sort: z.enum(['started_at', 'updated_at']).meta({ description: 'Field to sort by' }).optional(),
|
|
1024
1382
|
order: z.enum(['asc', 'desc']).meta({ description: 'Sort order' }).optional(),
|
|
1383
|
+
evaluation_severity: z
|
|
1384
|
+
.array(ListAgentRunsEvaluationSeveritySchema)
|
|
1385
|
+
.meta({ description: 'Filter by evaluation severity; `unrated` selects runs without an evaluation' })
|
|
1386
|
+
.optional(),
|
|
1387
|
+
evaluation_flag: z
|
|
1388
|
+
.array(TurnEvaluationFlagSchema)
|
|
1389
|
+
.meta({ description: 'Filter by evaluation flag (any of)' })
|
|
1390
|
+
.optional(),
|
|
1391
|
+
feedback_rating: AgentRunFeedbackRatingSchema.meta({
|
|
1392
|
+
description: 'Filter by last feedback rating',
|
|
1393
|
+
}).optional(),
|
|
1394
|
+
contradicted: z
|
|
1395
|
+
.boolean()
|
|
1396
|
+
.meta({ description: 'Only runs whose feedback or judge contradicts the detectors' })
|
|
1397
|
+
.optional(),
|
|
1025
1398
|
})
|
|
1026
1399
|
.meta({ id: 'ListAgentRunsQuery' });
|
|
1027
1400
|
|
|
@@ -1107,6 +1480,7 @@ export const UpdateAgentRunStatusPayloadSchema = z
|
|
|
1107
1480
|
last_archive_error: z.string().optional(),
|
|
1108
1481
|
sequence: z.number().optional(),
|
|
1109
1482
|
process_state: ProcessStateSchema.optional(),
|
|
1483
|
+
evaluation_rollup: AgentRunEvaluationRollupSchema.optional(),
|
|
1110
1484
|
})
|
|
1111
1485
|
.meta({ id: 'UpdateAgentRunStatusPayload' });
|
|
1112
1486
|
|
|
@@ -28,6 +28,12 @@ describe('data-store API contracts', () => {
|
|
|
28
28
|
>();
|
|
29
29
|
});
|
|
30
30
|
|
|
31
|
+
it('accepts a recoverable import ID at the HTTP request boundary', () => {
|
|
32
|
+
const input = { mode: 'append', message: 'Import rows', tables: {}, import_id: '6aa400001234567890abcdef' };
|
|
33
|
+
expect(validateApiRequest('ImportDataPayload', input).valid).toBe(true);
|
|
34
|
+
expect(validateApiRequest('ImportDataPayload', { ...input, import_id: 'invalid' }).valid).toBe(false);
|
|
35
|
+
});
|
|
36
|
+
|
|
31
37
|
it('rejects undeclared request fields through the published component', () => {
|
|
32
38
|
expect(
|
|
33
39
|
validateApiRequest('CreateDataStorePayload', {
|
|
@@ -520,6 +520,14 @@ export const DataStoreVersionArraySchema = z.array(DataStoreVersionSchema).meta(
|
|
|
520
520
|
|
|
521
521
|
export const ImportDataPayloadSchema = z
|
|
522
522
|
.strictObject({
|
|
523
|
+
import_id: z
|
|
524
|
+
.string()
|
|
525
|
+
.regex(/^[0-9a-f]{24}$/)
|
|
526
|
+
.meta({
|
|
527
|
+
description:
|
|
528
|
+
'Optional client-generated Mongo ObjectId for idempotent retries. Generate once before submitting, then reuse with identical input. New IDs must be less than 24 hours old; existing jobs are returned without executing again. Poll GET /data/:storeId/import/:importId after a timeout.',
|
|
529
|
+
})
|
|
530
|
+
.optional(),
|
|
523
531
|
tables: ImportTableDataMapSchema.meta({ description: 'Map of table name to data specification' }),
|
|
524
532
|
mode: z.enum(['append', 'replace']).meta({ description: 'Import mode' }),
|
|
525
533
|
message: z.string().meta({ description: 'Commit message' }),
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
const id = z.string().regex(/^[a-f0-9]{24}$/i);
|
|
4
|
+
export const CreateDelegationGrantPayloadSchema = z
|
|
5
|
+
.strictObject({
|
|
6
|
+
schedule_id: id,
|
|
7
|
+
subject: id,
|
|
8
|
+
output_collection_id: id,
|
|
9
|
+
policy_hash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
10
|
+
expires_at: z.string().datetime().nullable(),
|
|
11
|
+
})
|
|
12
|
+
.meta({ id: 'CreateDelegationGrantPayload' });
|
|
13
|
+
export const DelegationGrantSchema = CreateDelegationGrantPayloadSchema.extend({
|
|
14
|
+
id,
|
|
15
|
+
account_id: id,
|
|
16
|
+
project_id: id,
|
|
17
|
+
granter: id,
|
|
18
|
+
created_at: z.string().datetime(),
|
|
19
|
+
revoked_at: z.string().datetime().nullable(),
|
|
20
|
+
}).meta({ id: 'DelegationGrant' });
|
|
21
|
+
export const DelegationGrantArraySchema = z.array(DelegationGrantSchema).meta({ id: 'DelegationGrantArray' });
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { toOpenApiComponents } from './adapter.js';
|
|
4
|
+
import { emitJsonSchema } from './emit-json-schema.js';
|
|
5
|
+
|
|
6
|
+
describe('explicit union emission', () => {
|
|
7
|
+
it('preserves optional explicit unions and nullable schemas', () => {
|
|
8
|
+
const schema = z.object({
|
|
9
|
+
explicit: z.union([z.string(), z.null()]).optional().describe('Clear with null'),
|
|
10
|
+
nullable: z.string().nullable(),
|
|
11
|
+
});
|
|
12
|
+
const emitted = emitJsonSchema(schema);
|
|
13
|
+
expect(emitted.properties).toEqual({
|
|
14
|
+
explicit: { anyOf: [{ type: 'string' }, { type: 'null' }], description: 'Clear with null' },
|
|
15
|
+
nullable: { anyOf: [{ type: 'string' }, { type: 'null' }] },
|
|
16
|
+
});
|
|
17
|
+
expect(JSON.stringify(emitted)).not.toContain('x-vertesia-explicit-union');
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('preserves referenced explicit unions inside named definitions', () => {
|
|
21
|
+
const value = z.union([z.string(), z.boolean(), z.number(), z.null()]).meta({ id: 'Value' });
|
|
22
|
+
const schema = z.object({ value }).meta({ id: 'Holder' });
|
|
23
|
+
const components = toOpenApiComponents({ Holder: emitJsonSchema(schema) });
|
|
24
|
+
expect(components.Value).toEqual({
|
|
25
|
+
anyOf: [{ type: 'string' }, { type: 'boolean' }, { type: 'number' }, { type: 'null' }],
|
|
26
|
+
});
|
|
27
|
+
expect(JSON.stringify(components)).not.toContain('x-vertesia-explicit-union');
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('retains constraints on union branches', () => {
|
|
31
|
+
expect(emitJsonSchema(z.union([z.string().min(2), z.null()])).anyOf).toEqual([
|
|
32
|
+
{ type: 'string', minLength: 2 },
|
|
33
|
+
{ type: 'null' },
|
|
34
|
+
]);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { isPlainObject, type JsonObject } from './adapter.js';
|
|
3
|
+
|
|
4
|
+
const EXPLICIT_UNION = 'x-vertesia-explicit-union';
|
|
5
|
+
|
|
6
|
+
/** Preserve the published explicit-union spelling across Zod's final compaction pass. */
|
|
7
|
+
export function emitJsonSchema(schema: z.ZodType): JsonObject {
|
|
8
|
+
const emitted = z.toJSONSchema(schema, {
|
|
9
|
+
target: 'draft-2020-12',
|
|
10
|
+
io: 'input',
|
|
11
|
+
override: ({ jsonSchema }) => {
|
|
12
|
+
// Zod 4.5 compacts bare anyOf branches AFTER override runs. Carry the original
|
|
13
|
+
// branches through that pass; optional wrappers inherit this metadata as well.
|
|
14
|
+
const branches = jsonSchema.anyOf;
|
|
15
|
+
if (
|
|
16
|
+
branches?.every(
|
|
17
|
+
(branch) => isPlainObject(branch) && Object.keys(branch).length === 1 && 'type' in branch,
|
|
18
|
+
)
|
|
19
|
+
) {
|
|
20
|
+
jsonSchema[EXPLICIT_UNION] = branches;
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
});
|
|
24
|
+
restoreExplicitUnions(emitted);
|
|
25
|
+
return emitted;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function restoreExplicitUnions(value: unknown): void {
|
|
29
|
+
if (Array.isArray(value)) {
|
|
30
|
+
for (const child of value) restoreExplicitUnions(child);
|
|
31
|
+
} else if (isPlainObject(value)) {
|
|
32
|
+
const branches = value[EXPLICIT_UNION];
|
|
33
|
+
if (Array.isArray(branches)) {
|
|
34
|
+
delete value[EXPLICIT_UNION];
|
|
35
|
+
delete value.type;
|
|
36
|
+
value.anyOf = branches;
|
|
37
|
+
}
|
|
38
|
+
for (const child of Object.values(value)) restoreExplicitUnions(child);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -32,6 +32,9 @@ describe('the provider vocabulary', () => {
|
|
|
32
32
|
expect(validateApiRequest('ExecutionEnvironmentCreatePayload', { name: 'e', provider: 'openai' }).valid).toBe(
|
|
33
33
|
true,
|
|
34
34
|
);
|
|
35
|
+
expect(
|
|
36
|
+
validateApiRequest('ExecutionEnvironmentCreatePayload', { name: 'router', provider: 'openrouter' }).valid,
|
|
37
|
+
).toBe(true);
|
|
35
38
|
expect(validateApiRequest('ExecutionEnvironmentCreatePayload', { name: 'e', provider: 'gpt' }).valid).toBe(
|
|
36
39
|
false,
|
|
37
40
|
);
|
package/src/api-schemas/index.ts
CHANGED
|
@@ -31,6 +31,7 @@ export * from './integrations.js';
|
|
|
31
31
|
export * from './oauth-server.js';
|
|
32
32
|
export * from './parameters.js';
|
|
33
33
|
export * from './process.js';
|
|
34
|
+
export * from './process-agent-policy.js';
|
|
34
35
|
export * from './quota.js';
|
|
35
36
|
export * from './registry.js';
|
|
36
37
|
export * from './secrets.js';
|
|
@@ -93,6 +93,29 @@ describe('AsyncConversationExecutionPayload contract', () => {
|
|
|
93
93
|
});
|
|
94
94
|
});
|
|
95
95
|
|
|
96
|
+
it('accepts Process-authored agent execution policy', () => {
|
|
97
|
+
const payload: AsyncConversationExecutionPayload = {
|
|
98
|
+
type: 'conversation',
|
|
99
|
+
interaction: 'sys:AppDeveloper',
|
|
100
|
+
agent_policy: {
|
|
101
|
+
phases: [
|
|
102
|
+
{
|
|
103
|
+
id: 'saved',
|
|
104
|
+
tools: ['app_workspace_save'],
|
|
105
|
+
tool_input_contains: [{ field: 'mode', contains: 'commit' }],
|
|
106
|
+
recovery_prompt: 'Save before finishing.',
|
|
107
|
+
},
|
|
108
|
+
],
|
|
109
|
+
phase_resets: [{ tools: ['app_workspace_edit'], to_phase: 'saved' }],
|
|
110
|
+
action_phase_count: 1,
|
|
111
|
+
action_phase_max_tokens: 4_096,
|
|
112
|
+
completion_prompt: 'Return the structured result now.',
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
expect(AsyncConversationExecutionPayloadSchema.parse(payload)).toMatchObject(payload);
|
|
117
|
+
});
|
|
118
|
+
|
|
96
119
|
it('rejects a non-string app-version target', () => {
|
|
97
120
|
expect(() =>
|
|
98
121
|
AsyncConversationExecutionPayloadSchema.parse({
|