@dotsetlabs/bellwether 1.0.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +8 -2
- package/dist/baseline/accessors.d.ts +1 -1
- package/dist/baseline/accessors.js +1 -3
- package/dist/baseline/baseline-format.d.ts +287 -0
- package/dist/baseline/baseline-format.js +12 -0
- package/dist/baseline/comparator.js +249 -11
- package/dist/baseline/converter.d.ts +15 -15
- package/dist/baseline/converter.js +46 -34
- package/dist/baseline/diff.d.ts +1 -1
- package/dist/baseline/diff.js +45 -28
- package/dist/baseline/error-analyzer.d.ts +1 -1
- package/dist/baseline/error-analyzer.js +90 -17
- package/dist/baseline/incremental-checker.js +8 -5
- package/dist/baseline/index.d.ts +2 -12
- package/dist/baseline/index.js +3 -23
- package/dist/baseline/performance-tracker.d.ts +0 -1
- package/dist/baseline/performance-tracker.js +13 -20
- package/dist/baseline/response-fingerprint.js +39 -2
- package/dist/baseline/saver.js +41 -10
- package/dist/baseline/schema-compare.d.ts +22 -0
- package/dist/baseline/schema-compare.js +259 -16
- package/dist/baseline/types.d.ts +10 -7
- package/dist/cache/response-cache.d.ts +8 -0
- package/dist/cache/response-cache.js +110 -0
- package/dist/cli/commands/check.js +23 -6
- package/dist/cli/commands/explore.js +34 -14
- package/dist/cli/index.js +8 -0
- package/dist/config/template.js +8 -7
- package/dist/config/validator.d.ts +59 -59
- package/dist/config/validator.js +245 -90
- package/dist/constants/core.d.ts +4 -0
- package/dist/constants/core.js +8 -19
- package/dist/constants/registry.d.ts +17 -0
- package/dist/constants/registry.js +18 -0
- package/dist/constants/testing.d.ts +0 -369
- package/dist/constants/testing.js +18 -456
- package/dist/constants.d.ts +1 -1
- package/dist/constants.js +1 -1
- package/dist/docs/contract.js +131 -83
- package/dist/docs/report.js +8 -5
- package/dist/interview/insights.d.ts +17 -0
- package/dist/interview/insights.js +52 -0
- package/dist/interview/interviewer.js +52 -10
- package/dist/interview/prompt-test-generator.d.ts +12 -0
- package/dist/interview/prompt-test-generator.js +77 -0
- package/dist/interview/resource-test-generator.d.ts +12 -0
- package/dist/interview/resource-test-generator.js +20 -0
- package/dist/interview/schema-inferrer.js +26 -4
- package/dist/interview/schema-test-generator.js +278 -31
- package/dist/interview/stateful-test-runner.d.ts +3 -0
- package/dist/interview/stateful-test-runner.js +80 -0
- package/dist/interview/types.d.ts +12 -0
- package/dist/transport/mcp-client.js +1 -1
- package/dist/transport/sse-transport.d.ts +7 -3
- package/dist/transport/sse-transport.js +157 -67
- package/dist/version.js +1 -1
- package/man/bellwether.1 +1 -1
- package/man/bellwether.1.md +2 -2
- package/package.json +1 -1
- package/schemas/bellwether-check.schema.json +185 -0
- package/schemas/bellwether-explore.schema.json +837 -0
- package/scripts/completions/bellwether.bash +10 -4
- package/scripts/completions/bellwether.zsh +55 -2
package/dist/config/validator.js
CHANGED
|
@@ -13,7 +13,8 @@ import { getExternalServiceStatus } from '../baseline/external-dependency-detect
|
|
|
13
13
|
/**
|
|
14
14
|
* Server configuration schema.
|
|
15
15
|
*/
|
|
16
|
-
export const serverConfigSchema = z
|
|
16
|
+
export const serverConfigSchema = z
|
|
17
|
+
.object({
|
|
17
18
|
/** Command to start the MCP server */
|
|
18
19
|
command: z.string().default(CONFIG_DEFAULTS.server.command),
|
|
19
20
|
/** Arguments to pass to the server command */
|
|
@@ -26,25 +27,31 @@ export const serverConfigSchema = z.object({
|
|
|
26
27
|
.max(VALIDATION_BOUNDS.TIMEOUT.MAX_MS)
|
|
27
28
|
.default(CONFIG_DEFAULTS.server.timeout),
|
|
28
29
|
/** Transport type */
|
|
29
|
-
transport: z
|
|
30
|
+
transport: z
|
|
31
|
+
.enum(['stdio', 'sse', 'streamable-http'])
|
|
32
|
+
.default(CONFIG_DEFAULTS.server.transport),
|
|
30
33
|
/** Remote MCP server URL (required for sse/streamable-http) */
|
|
31
34
|
url: z.string().optional().default(CONFIG_DEFAULTS.server.url),
|
|
32
35
|
/** Session ID for remote auth */
|
|
33
36
|
sessionId: z.string().optional().default(CONFIG_DEFAULTS.server.sessionId),
|
|
34
37
|
/** Additional environment variables */
|
|
35
38
|
env: z.record(z.string()).optional(),
|
|
36
|
-
})
|
|
39
|
+
})
|
|
40
|
+
.default(CONFIG_DEFAULTS.server);
|
|
37
41
|
/**
|
|
38
42
|
* LLM Ollama-specific settings.
|
|
39
43
|
*/
|
|
40
|
-
export const ollamaConfigSchema = z
|
|
44
|
+
export const ollamaConfigSchema = z
|
|
45
|
+
.object({
|
|
41
46
|
/** Ollama server base URL */
|
|
42
47
|
baseUrl: z.string().url().default(CONFIG_DEFAULTS.llm.ollama.baseUrl),
|
|
43
|
-
})
|
|
48
|
+
})
|
|
49
|
+
.default(CONFIG_DEFAULTS.llm.ollama);
|
|
44
50
|
/**
|
|
45
51
|
* LLM configuration schema.
|
|
46
52
|
*/
|
|
47
|
-
export const llmConfigSchema = z
|
|
53
|
+
export const llmConfigSchema = z
|
|
54
|
+
.object({
|
|
48
55
|
/** LLM provider */
|
|
49
56
|
provider: z.enum(['ollama', 'openai', 'anthropic']).default(CONFIG_DEFAULTS.llm.provider),
|
|
50
57
|
/** Model to use (empty = provider default) */
|
|
@@ -55,18 +62,17 @@ export const llmConfigSchema = z.object({
|
|
|
55
62
|
openaiApiKeyEnvVar: z.string().optional(),
|
|
56
63
|
/** Environment variable for Anthropic API key */
|
|
57
64
|
anthropicApiKeyEnvVar: z.string().optional(),
|
|
58
|
-
})
|
|
65
|
+
})
|
|
66
|
+
.default(CONFIG_DEFAULTS.llm);
|
|
59
67
|
/**
|
|
60
68
|
* Explore configuration schema (for bellwether explore command).
|
|
61
69
|
*/
|
|
62
|
-
export const exploreConfigSchema = z
|
|
70
|
+
export const exploreConfigSchema = z
|
|
71
|
+
.object({
|
|
63
72
|
/** Personas to use for exploration */
|
|
64
|
-
personas: z
|
|
65
|
-
'technical_writer',
|
|
66
|
-
|
|
67
|
-
'qa_engineer',
|
|
68
|
-
'novice_user',
|
|
69
|
-
])).default([...CONFIG_DEFAULTS.explore.personas]),
|
|
73
|
+
personas: z
|
|
74
|
+
.array(z.enum(['technical_writer', 'security_tester', 'qa_engineer', 'novice_user']))
|
|
75
|
+
.default([...CONFIG_DEFAULTS.explore.personas]),
|
|
70
76
|
/** Maximum questions per tool */
|
|
71
77
|
maxQuestionsPerTool: z
|
|
72
78
|
.number()
|
|
@@ -85,7 +91,8 @@ export const exploreConfigSchema = z.object({
|
|
|
85
91
|
.default(CONFIG_DEFAULTS.explore.personaConcurrency),
|
|
86
92
|
/** Skip error/edge case testing */
|
|
87
93
|
skipErrorTests: z.boolean().default(CONFIG_DEFAULTS.explore.skipErrorTests),
|
|
88
|
-
})
|
|
94
|
+
})
|
|
95
|
+
.default(() => ({
|
|
89
96
|
personas: [...CONFIG_DEFAULTS.explore.personas],
|
|
90
97
|
maxQuestionsPerTool: CONFIG_DEFAULTS.explore.maxQuestionsPerTool,
|
|
91
98
|
parallelPersonas: CONFIG_DEFAULTS.explore.parallelPersonas,
|
|
@@ -95,16 +102,19 @@ export const exploreConfigSchema = z.object({
|
|
|
95
102
|
/**
|
|
96
103
|
* Scenarios configuration schema.
|
|
97
104
|
*/
|
|
98
|
-
export const scenariosConfigSchema = z
|
|
105
|
+
export const scenariosConfigSchema = z
|
|
106
|
+
.object({
|
|
99
107
|
/** Path to scenarios YAML file */
|
|
100
108
|
path: z.string().optional(),
|
|
101
109
|
/** Run only scenarios (no LLM tests) */
|
|
102
110
|
only: z.boolean().default(CONFIG_DEFAULTS.scenarios.only),
|
|
103
|
-
})
|
|
111
|
+
})
|
|
112
|
+
.default(CONFIG_DEFAULTS.scenarios);
|
|
104
113
|
/**
|
|
105
114
|
* Workflows configuration schema.
|
|
106
115
|
*/
|
|
107
|
-
export const workflowsConfigSchema = z
|
|
116
|
+
export const workflowsConfigSchema = z
|
|
117
|
+
.object({
|
|
108
118
|
/** Path to workflows YAML file */
|
|
109
119
|
path: z.string().optional(),
|
|
110
120
|
/** Enable LLM-based workflow discovery */
|
|
@@ -114,33 +124,76 @@ export const workflowsConfigSchema = z.object({
|
|
|
114
124
|
/** Auto-generate workflows from discovered tools */
|
|
115
125
|
autoGenerate: z.boolean().default(CONFIG_DEFAULTS.workflows.autoGenerate),
|
|
116
126
|
/** Skip steps whose dependencies (previous steps providing data) have failed */
|
|
117
|
-
requireSuccessfulDependencies: z
|
|
127
|
+
requireSuccessfulDependencies: z
|
|
128
|
+
.boolean()
|
|
129
|
+
.default(CONFIG_DEFAULTS.workflows.requireSuccessfulDependencies),
|
|
118
130
|
/** Timeout per workflow step in milliseconds */
|
|
119
|
-
stepTimeout: z
|
|
131
|
+
stepTimeout: z
|
|
132
|
+
.number()
|
|
133
|
+
.int()
|
|
134
|
+
.min(1000)
|
|
135
|
+
.max(300000)
|
|
136
|
+
.default(CONFIG_DEFAULTS.workflows.stepTimeout),
|
|
120
137
|
/** Timeout configuration for workflow operations */
|
|
121
|
-
timeouts: z
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
138
|
+
timeouts: z
|
|
139
|
+
.object({
|
|
140
|
+
toolCall: z
|
|
141
|
+
.number()
|
|
142
|
+
.int()
|
|
143
|
+
.min(1000)
|
|
144
|
+
.max(300000)
|
|
145
|
+
.default(CONFIG_DEFAULTS.workflows.timeouts.toolCall),
|
|
146
|
+
stateSnapshot: z
|
|
147
|
+
.number()
|
|
148
|
+
.int()
|
|
149
|
+
.min(1000)
|
|
150
|
+
.max(300000)
|
|
151
|
+
.default(CONFIG_DEFAULTS.workflows.timeouts.stateSnapshot),
|
|
152
|
+
probeTool: z
|
|
153
|
+
.number()
|
|
154
|
+
.int()
|
|
155
|
+
.min(1000)
|
|
156
|
+
.max(300000)
|
|
157
|
+
.default(CONFIG_DEFAULTS.workflows.timeouts.probeTool),
|
|
158
|
+
llmAnalysis: z
|
|
159
|
+
.number()
|
|
160
|
+
.int()
|
|
161
|
+
.min(1000)
|
|
162
|
+
.max(300000)
|
|
163
|
+
.default(CONFIG_DEFAULTS.workflows.timeouts.llmAnalysis),
|
|
164
|
+
llmSummary: z
|
|
165
|
+
.number()
|
|
166
|
+
.int()
|
|
167
|
+
.min(1000)
|
|
168
|
+
.max(300000)
|
|
169
|
+
.default(CONFIG_DEFAULTS.workflows.timeouts.llmSummary),
|
|
170
|
+
})
|
|
171
|
+
.default(CONFIG_DEFAULTS.workflows.timeouts),
|
|
172
|
+
})
|
|
173
|
+
.default(CONFIG_DEFAULTS.workflows);
|
|
129
174
|
/**
|
|
130
175
|
* Example output configuration schema.
|
|
131
176
|
*/
|
|
132
|
-
export const examplesConfigSchema = z
|
|
177
|
+
export const examplesConfigSchema = z
|
|
178
|
+
.object({
|
|
133
179
|
/** Include full (non-truncated) examples */
|
|
134
180
|
full: z.boolean().default(CONFIG_DEFAULTS.output.examples.full),
|
|
135
181
|
/** Maximum example length in characters */
|
|
136
|
-
maxLength: z
|
|
182
|
+
maxLength: z
|
|
183
|
+
.number()
|
|
184
|
+
.int()
|
|
185
|
+
.min(100)
|
|
186
|
+
.max(50000)
|
|
187
|
+
.default(CONFIG_DEFAULTS.output.examples.maxLength),
|
|
137
188
|
/** Maximum examples per tool */
|
|
138
189
|
maxPerTool: z.number().int().min(1).max(20).default(CONFIG_DEFAULTS.output.examples.maxPerTool),
|
|
139
|
-
})
|
|
190
|
+
})
|
|
191
|
+
.default(CONFIG_DEFAULTS.output.examples);
|
|
140
192
|
/**
|
|
141
193
|
* Output file name configuration schema.
|
|
142
194
|
*/
|
|
143
|
-
export const outputFilesConfigSchema = z
|
|
195
|
+
export const outputFilesConfigSchema = z
|
|
196
|
+
.object({
|
|
144
197
|
/** Check report JSON file name */
|
|
145
198
|
checkReport: z.string().default(CONFIG_DEFAULTS.output.files.checkReport),
|
|
146
199
|
/** Explore report JSON file name */
|
|
@@ -149,22 +202,38 @@ export const outputFilesConfigSchema = z.object({
|
|
|
149
202
|
contractDoc: z.string().default(CONFIG_DEFAULTS.output.files.contractDoc),
|
|
150
203
|
/** Agents documentation file name */
|
|
151
204
|
agentsDoc: z.string().default(CONFIG_DEFAULTS.output.files.agentsDoc),
|
|
152
|
-
})
|
|
205
|
+
})
|
|
206
|
+
.default(CONFIG_DEFAULTS.output.files);
|
|
207
|
+
/**
|
|
208
|
+
* Output format schema.
|
|
209
|
+
* Accepts legacy values (agents.md) and synonyms (docs, markdown) and normalizes to "docs".
|
|
210
|
+
*/
|
|
211
|
+
const outputFormatSchema = z.preprocess((value) => {
|
|
212
|
+
if (typeof value === 'string') {
|
|
213
|
+
const normalized = value.trim().toLowerCase();
|
|
214
|
+
if (normalized === 'agents.md' || normalized === 'markdown' || normalized === 'docs') {
|
|
215
|
+
return 'docs';
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return value;
|
|
219
|
+
}, z.enum(['docs', 'json', 'both']));
|
|
153
220
|
/**
|
|
154
221
|
* Output configuration schema.
|
|
155
222
|
*/
|
|
156
|
-
export const outputConfigSchema = z
|
|
223
|
+
export const outputConfigSchema = z
|
|
224
|
+
.object({
|
|
157
225
|
/** Output directory for JSON files (bellwether-check.json, etc.) */
|
|
158
226
|
dir: z.string().default(CONFIG_DEFAULTS.output.dir),
|
|
159
227
|
/** Output directory for documentation files (CONTRACT.md, AGENTS.md) */
|
|
160
228
|
docsDir: z.string().default(CONFIG_DEFAULTS.output.docsDir),
|
|
161
229
|
/** Output format */
|
|
162
|
-
format:
|
|
230
|
+
format: outputFormatSchema.default(CONFIG_DEFAULTS.output.format),
|
|
163
231
|
/** Example output settings */
|
|
164
232
|
examples: examplesConfigSchema,
|
|
165
233
|
/** Output file names */
|
|
166
234
|
files: outputFilesConfigSchema,
|
|
167
|
-
})
|
|
235
|
+
})
|
|
236
|
+
.default(CONFIG_DEFAULTS.output);
|
|
168
237
|
/**
|
|
169
238
|
* Severity levels for configuration.
|
|
170
239
|
*/
|
|
@@ -175,70 +244,96 @@ const severityLevels = ['none', 'info', 'warning', 'breaking'];
|
|
|
175
244
|
const behaviorAspects = [
|
|
176
245
|
'response_format',
|
|
177
246
|
'response_structure',
|
|
247
|
+
'response_schema_evolution',
|
|
178
248
|
'error_handling',
|
|
179
249
|
'error_pattern',
|
|
180
250
|
'security',
|
|
181
251
|
'performance',
|
|
182
252
|
'schema',
|
|
183
253
|
'description',
|
|
254
|
+
'prompt',
|
|
255
|
+
'resource',
|
|
256
|
+
'server',
|
|
257
|
+
'capability',
|
|
184
258
|
];
|
|
185
259
|
/**
|
|
186
260
|
* Severity configuration schema.
|
|
187
261
|
* Allows customizing how changes are classified and reported.
|
|
188
262
|
*/
|
|
189
|
-
export const severityConfigSchema = z
|
|
263
|
+
export const severityConfigSchema = z
|
|
264
|
+
.object({
|
|
190
265
|
/** Minimum severity level to include in reports */
|
|
191
|
-
minimumSeverity: z
|
|
266
|
+
minimumSeverity: z
|
|
267
|
+
.enum(severityLevels)
|
|
268
|
+
.default(CONFIG_DEFAULTS.baseline.severity.minimumSeverity),
|
|
192
269
|
/** Severity level at which to fail CI checks */
|
|
193
|
-
failOnSeverity: z
|
|
270
|
+
failOnSeverity: z
|
|
271
|
+
.enum(severityLevels)
|
|
272
|
+
.default(CONFIG_DEFAULTS.baseline.severity.failOnSeverity),
|
|
194
273
|
/** Suppress warning-level changes from output */
|
|
195
274
|
suppressWarnings: z.boolean().default(CONFIG_DEFAULTS.baseline.severity.suppressWarnings),
|
|
196
275
|
/** Custom severity overrides per aspect */
|
|
197
276
|
aspectOverrides: z.record(z.enum(behaviorAspects), z.enum(severityLevels)).optional(),
|
|
198
|
-
})
|
|
277
|
+
})
|
|
278
|
+
.default(CONFIG_DEFAULTS.baseline.severity);
|
|
199
279
|
/**
|
|
200
280
|
* Security testing configuration schema.
|
|
201
281
|
*/
|
|
202
|
-
export const securityConfigSchema = z
|
|
282
|
+
export const securityConfigSchema = z
|
|
283
|
+
.object({
|
|
203
284
|
/** Enable security vulnerability testing */
|
|
204
285
|
enabled: z.boolean().default(CONFIG_DEFAULTS.check.security.enabled),
|
|
205
286
|
/** Security categories to test */
|
|
206
|
-
categories: z
|
|
287
|
+
categories: z
|
|
288
|
+
.array(z.enum([
|
|
207
289
|
'sql_injection',
|
|
208
290
|
'xss',
|
|
209
291
|
'path_traversal',
|
|
210
292
|
'command_injection',
|
|
211
293
|
'ssrf',
|
|
212
294
|
'error_disclosure',
|
|
213
|
-
]))
|
|
214
|
-
|
|
295
|
+
]))
|
|
296
|
+
.default([...CONFIG_DEFAULTS.check.security.categories]),
|
|
297
|
+
})
|
|
298
|
+
.default(() => ({
|
|
215
299
|
enabled: CONFIG_DEFAULTS.check.security.enabled,
|
|
216
300
|
categories: [...CONFIG_DEFAULTS.check.security.categories],
|
|
217
301
|
}));
|
|
218
302
|
/**
|
|
219
303
|
* Statistical sampling configuration schema.
|
|
220
304
|
*/
|
|
221
|
-
export const samplingConfigSchema = z
|
|
305
|
+
export const samplingConfigSchema = z
|
|
306
|
+
.object({
|
|
222
307
|
/** Minimum samples per tool for statistical confidence */
|
|
223
308
|
minSamples: z.number().int().min(1).max(50).default(CONFIG_DEFAULTS.check.sampling.minSamples),
|
|
224
309
|
/** Target confidence level */
|
|
225
|
-
targetConfidence: z
|
|
310
|
+
targetConfidence: z
|
|
311
|
+
.enum(['low', 'medium', 'high'])
|
|
312
|
+
.default(CONFIG_DEFAULTS.check.sampling.targetConfidence),
|
|
226
313
|
/** Fail if confidence is below target */
|
|
227
314
|
failOnLowConfidence: z.boolean().default(CONFIG_DEFAULTS.check.sampling.failOnLowConfidence),
|
|
228
|
-
})
|
|
315
|
+
})
|
|
316
|
+
.default(CONFIG_DEFAULTS.check.sampling);
|
|
229
317
|
/**
|
|
230
318
|
* Metrics configuration schema.
|
|
231
319
|
*/
|
|
232
|
-
export const metricsConfigSchema = z
|
|
320
|
+
export const metricsConfigSchema = z
|
|
321
|
+
.object({
|
|
233
322
|
/** Count validation rejections as success */
|
|
234
|
-
countValidationAsSuccess: z
|
|
323
|
+
countValidationAsSuccess: z
|
|
324
|
+
.boolean()
|
|
325
|
+
.default(CONFIG_DEFAULTS.check.metrics.countValidationAsSuccess),
|
|
235
326
|
/** Separate validation metrics from reliability metrics */
|
|
236
|
-
separateValidationMetrics: z
|
|
237
|
-
|
|
327
|
+
separateValidationMetrics: z
|
|
328
|
+
.boolean()
|
|
329
|
+
.default(CONFIG_DEFAULTS.check.metrics.separateValidationMetrics),
|
|
330
|
+
})
|
|
331
|
+
.default(CONFIG_DEFAULTS.check.metrics);
|
|
238
332
|
/**
|
|
239
333
|
* Stateful testing configuration schema.
|
|
240
334
|
*/
|
|
241
|
-
export const statefulTestingConfigSchema = z
|
|
335
|
+
export const statefulTestingConfigSchema = z
|
|
336
|
+
.object({
|
|
242
337
|
/** Enable stateful tool testing */
|
|
243
338
|
enabled: z.boolean().default(CONFIG_DEFAULTS.check.statefulTesting.enabled),
|
|
244
339
|
/** Maximum dependency chain length */
|
|
@@ -249,8 +344,11 @@ export const statefulTestingConfigSchema = z.object({
|
|
|
249
344
|
.max(VALIDATION_BOUNDS.STATEFUL_CHAIN.MAX)
|
|
250
345
|
.default(CONFIG_DEFAULTS.check.statefulTesting.maxChainLength),
|
|
251
346
|
/** Share outputs between dependent tools */
|
|
252
|
-
shareOutputsBetweenTools: z
|
|
253
|
-
|
|
347
|
+
shareOutputsBetweenTools: z
|
|
348
|
+
.boolean()
|
|
349
|
+
.default(CONFIG_DEFAULTS.check.statefulTesting.shareOutputsBetweenTools),
|
|
350
|
+
})
|
|
351
|
+
.default(CONFIG_DEFAULTS.check.statefulTesting);
|
|
254
352
|
/**
|
|
255
353
|
* External service configuration schema.
|
|
256
354
|
*/
|
|
@@ -263,27 +361,32 @@ export const externalServiceConfigSchema = z.object({
|
|
|
263
361
|
/**
|
|
264
362
|
* External services handling configuration schema.
|
|
265
363
|
*/
|
|
266
|
-
export const externalServicesConfigSchema = z
|
|
364
|
+
export const externalServicesConfigSchema = z
|
|
365
|
+
.object({
|
|
267
366
|
/** Mode for unconfigured services */
|
|
268
367
|
mode: z.enum(['skip', 'mock', 'fail']).default(CONFIG_DEFAULTS.check.externalServices.mode),
|
|
269
368
|
/** Per-service configuration overrides */
|
|
270
369
|
services: z.record(externalServiceConfigSchema).optional(),
|
|
271
|
-
})
|
|
370
|
+
})
|
|
371
|
+
.default(CONFIG_DEFAULTS.check.externalServices);
|
|
272
372
|
/**
|
|
273
373
|
* Response assertion configuration schema.
|
|
274
374
|
*/
|
|
275
|
-
export const assertionsConfigSchema = z
|
|
375
|
+
export const assertionsConfigSchema = z
|
|
376
|
+
.object({
|
|
276
377
|
/** Enable response assertions */
|
|
277
378
|
enabled: z.boolean().default(CONFIG_DEFAULTS.check.assertions.enabled),
|
|
278
379
|
/** Strict mode fails on assertion violations */
|
|
279
380
|
strict: z.boolean().default(CONFIG_DEFAULTS.check.assertions.strict),
|
|
280
381
|
/** Infer schemas from responses */
|
|
281
382
|
infer: z.boolean().default(CONFIG_DEFAULTS.check.assertions.infer),
|
|
282
|
-
})
|
|
383
|
+
})
|
|
384
|
+
.default(CONFIG_DEFAULTS.check.assertions);
|
|
283
385
|
/**
|
|
284
386
|
* Rate limiting configuration schema.
|
|
285
387
|
*/
|
|
286
|
-
export const rateLimitConfigSchema = z
|
|
388
|
+
export const rateLimitConfigSchema = z
|
|
389
|
+
.object({
|
|
287
390
|
/** Enable rate limiting */
|
|
288
391
|
enabled: z.boolean().default(CONFIG_DEFAULTS.check.rateLimit.enabled),
|
|
289
392
|
/** Requests per second */
|
|
@@ -300,7 +403,9 @@ export const rateLimitConfigSchema = z.object({
|
|
|
300
403
|
.max(VALIDATION_BOUNDS.RATE_LIMIT.BURST_LIMIT.MAX)
|
|
301
404
|
.default(CONFIG_DEFAULTS.check.rateLimit.burstLimit),
|
|
302
405
|
/** Backoff strategy */
|
|
303
|
-
backoffStrategy: z
|
|
406
|
+
backoffStrategy: z
|
|
407
|
+
.enum(['linear', 'exponential'])
|
|
408
|
+
.default(CONFIG_DEFAULTS.check.rateLimit.backoffStrategy),
|
|
304
409
|
/** Maximum retries on rate limit */
|
|
305
410
|
maxRetries: z
|
|
306
411
|
.number()
|
|
@@ -308,7 +413,8 @@ export const rateLimitConfigSchema = z.object({
|
|
|
308
413
|
.min(VALIDATION_BOUNDS.RATE_LIMIT.MAX_RETRIES.MIN)
|
|
309
414
|
.max(VALIDATION_BOUNDS.RATE_LIMIT.MAX_RETRIES.MAX)
|
|
310
415
|
.default(CONFIG_DEFAULTS.check.rateLimit.maxRetries),
|
|
311
|
-
})
|
|
416
|
+
})
|
|
417
|
+
.default(CONFIG_DEFAULTS.check.rateLimit);
|
|
312
418
|
/**
|
|
313
419
|
* Test fixture pattern schema.
|
|
314
420
|
* Allows matching parameter names by regex pattern.
|
|
@@ -323,29 +429,43 @@ export const testFixturePatternSchema = z.object({
|
|
|
323
429
|
* Test fixtures configuration schema.
|
|
324
430
|
* Allows users to customize test values for production server testing.
|
|
325
431
|
*/
|
|
326
|
-
export const testFixturesConfigSchema = z
|
|
432
|
+
export const testFixturesConfigSchema = z
|
|
433
|
+
.object({
|
|
327
434
|
/** Custom values for specific parameter names (exact match) */
|
|
328
435
|
parameterValues: z.record(z.unknown()).optional(),
|
|
329
436
|
/** Custom values for parameters matching regex patterns */
|
|
330
437
|
patterns: z.array(testFixturePatternSchema).optional(),
|
|
331
|
-
})
|
|
438
|
+
})
|
|
439
|
+
.optional();
|
|
332
440
|
/**
|
|
333
441
|
* Check command configuration schema.
|
|
334
442
|
* Controls behavior of `bellwether check`.
|
|
335
443
|
*/
|
|
336
|
-
export const checkConfigSchema = z
|
|
444
|
+
export const checkConfigSchema = z
|
|
445
|
+
.object({
|
|
337
446
|
/** Enable incremental checking (only test tools with changed schemas) */
|
|
338
447
|
incremental: z.boolean().default(CONFIG_DEFAULTS.check.incremental),
|
|
339
448
|
/** Maximum age of cached results in hours (for incremental checking) */
|
|
340
|
-
incrementalCacheHours: z
|
|
449
|
+
incrementalCacheHours: z
|
|
450
|
+
.number()
|
|
451
|
+
.int()
|
|
452
|
+
.min(1)
|
|
453
|
+
.max(720)
|
|
454
|
+
.default(CONFIG_DEFAULTS.check.incrementalCacheHours),
|
|
341
455
|
/** Enable parallel tool testing (faster checks) */
|
|
342
456
|
parallel: z.boolean().default(CONFIG_DEFAULTS.check.parallel),
|
|
343
457
|
/** Number of concurrent tool workers (1-10) */
|
|
344
458
|
parallelWorkers: z.number().int().min(1).max(10).default(CONFIG_DEFAULTS.check.parallelWorkers),
|
|
345
459
|
/** Performance regression threshold percentage (0-100, e.g., 25 = 25% slower triggers warning) */
|
|
346
|
-
performanceThreshold: z
|
|
460
|
+
performanceThreshold: z
|
|
461
|
+
.number()
|
|
462
|
+
.min(0)
|
|
463
|
+
.max(100)
|
|
464
|
+
.default(CONFIG_DEFAULTS.check.performanceThreshold),
|
|
347
465
|
/** Default diff output format */
|
|
348
|
-
diffFormat: z
|
|
466
|
+
diffFormat: z
|
|
467
|
+
.enum(['text', 'json', 'compact', 'github', 'markdown', 'junit', 'sarif'])
|
|
468
|
+
.default(CONFIG_DEFAULTS.check.diffFormat),
|
|
349
469
|
/** Number of warmup runs before timing samples (excluded from variance calculation) */
|
|
350
470
|
warmupRuns: z.number().int().min(0).max(5).default(CONFIG_DEFAULTS.check.warmupRuns),
|
|
351
471
|
/** Enable smart test value generation from schema descriptions (e.g., YYYY-MM-DD dates) */
|
|
@@ -366,7 +486,8 @@ export const checkConfigSchema = z.object({
|
|
|
366
486
|
sampling: samplingConfigSchema,
|
|
367
487
|
/** Test fixtures for production server testing */
|
|
368
488
|
testFixtures: testFixturesConfigSchema,
|
|
369
|
-
})
|
|
489
|
+
})
|
|
490
|
+
.default(() => ({
|
|
370
491
|
incremental: CONFIG_DEFAULTS.check.incremental,
|
|
371
492
|
incrementalCacheHours: CONFIG_DEFAULTS.check.incrementalCacheHours,
|
|
372
493
|
parallel: CONFIG_DEFAULTS.check.parallel,
|
|
@@ -389,7 +510,8 @@ export const checkConfigSchema = z.object({
|
|
|
389
510
|
/**
|
|
390
511
|
* Baseline configuration schema.
|
|
391
512
|
*/
|
|
392
|
-
export const baselineConfigSchema = z
|
|
513
|
+
export const baselineConfigSchema = z
|
|
514
|
+
.object({
|
|
393
515
|
/** Default baseline file path (relative to output.dir or absolute) */
|
|
394
516
|
path: z.string().default(CONFIG_DEFAULTS.baseline.path),
|
|
395
517
|
/** Path to save baseline after check (enables auto-save) */
|
|
@@ -399,32 +521,42 @@ export const baselineConfigSchema = z.object({
|
|
|
399
521
|
/** Fail if drift is detected */
|
|
400
522
|
failOnDrift: z.boolean().default(CONFIG_DEFAULTS.baseline.failOnDrift),
|
|
401
523
|
/** Default output format for baseline comparisons */
|
|
402
|
-
outputFormat: z
|
|
524
|
+
outputFormat: z
|
|
525
|
+
.enum(['text', 'json', 'markdown', 'compact'])
|
|
526
|
+
.default(CONFIG_DEFAULTS.baseline.outputFormat),
|
|
403
527
|
/** Severity thresholds for filtering and CI failure */
|
|
404
528
|
severity: severityConfigSchema,
|
|
405
|
-
})
|
|
529
|
+
})
|
|
530
|
+
.default(CONFIG_DEFAULTS.baseline);
|
|
406
531
|
/**
|
|
407
532
|
* Cache configuration schema.
|
|
408
533
|
*/
|
|
409
|
-
export const cacheConfigSchema = z
|
|
534
|
+
export const cacheConfigSchema = z
|
|
535
|
+
.object({
|
|
410
536
|
/** Enable response caching */
|
|
411
537
|
enabled: z.boolean().default(CONFIG_DEFAULTS.cache.enabled),
|
|
412
538
|
/** Cache directory */
|
|
413
539
|
dir: z.string().default(CONFIG_DEFAULTS.cache.dir),
|
|
414
|
-
})
|
|
540
|
+
})
|
|
541
|
+
.default(CONFIG_DEFAULTS.cache);
|
|
415
542
|
/**
|
|
416
543
|
* Logging configuration schema.
|
|
417
544
|
*/
|
|
418
|
-
export const loggingConfigSchema = z
|
|
545
|
+
export const loggingConfigSchema = z
|
|
546
|
+
.object({
|
|
419
547
|
/** Log level */
|
|
420
|
-
level: z
|
|
548
|
+
level: z
|
|
549
|
+
.enum(['debug', 'info', 'warn', 'error', 'silent'])
|
|
550
|
+
.default(CONFIG_DEFAULTS.logging.level),
|
|
421
551
|
/** Verbose output */
|
|
422
552
|
verbose: z.boolean().default(CONFIG_DEFAULTS.logging.verbose),
|
|
423
|
-
})
|
|
553
|
+
})
|
|
554
|
+
.default(CONFIG_DEFAULTS.logging);
|
|
424
555
|
/**
|
|
425
556
|
* Watch mode configuration schema.
|
|
426
557
|
*/
|
|
427
|
-
export const watchConfigSchema = z
|
|
558
|
+
export const watchConfigSchema = z
|
|
559
|
+
.object({
|
|
428
560
|
/** Path to watch for changes */
|
|
429
561
|
path: z.string().default(CONFIG_DEFAULTS.watch.path),
|
|
430
562
|
/** Polling interval in milliseconds */
|
|
@@ -433,7 +565,8 @@ export const watchConfigSchema = z.object({
|
|
|
433
565
|
extensions: z.array(z.string()).default([...CONFIG_DEFAULTS.watch.extensions]),
|
|
434
566
|
/** Command to run when drift is detected */
|
|
435
567
|
onDrift: z.string().optional(),
|
|
436
|
-
})
|
|
568
|
+
})
|
|
569
|
+
.default(() => ({
|
|
437
570
|
path: CONFIG_DEFAULTS.watch.path,
|
|
438
571
|
interval: CONFIG_DEFAULTS.watch.interval,
|
|
439
572
|
extensions: [...CONFIG_DEFAULTS.watch.extensions],
|
|
@@ -441,48 +574,64 @@ export const watchConfigSchema = z.object({
|
|
|
441
574
|
/**
|
|
442
575
|
* Discovery configuration schema.
|
|
443
576
|
*/
|
|
444
|
-
export const discoveryConfigSchema = z
|
|
577
|
+
export const discoveryConfigSchema = z
|
|
578
|
+
.object({
|
|
445
579
|
/** Output as JSON */
|
|
446
580
|
json: z.boolean().default(CONFIG_DEFAULTS.discovery.json),
|
|
447
581
|
/** Connection timeout in ms */
|
|
448
|
-
timeout: z
|
|
582
|
+
timeout: z
|
|
583
|
+
.number()
|
|
584
|
+
.int()
|
|
585
|
+
.min(VALIDATION_BOUNDS.TIMEOUT.MIN_MS)
|
|
586
|
+
.max(VALIDATION_BOUNDS.TIMEOUT.MAX_MS)
|
|
587
|
+
.default(CONFIG_DEFAULTS.discovery.timeout),
|
|
449
588
|
/** Transport type */
|
|
450
|
-
transport: z
|
|
589
|
+
transport: z
|
|
590
|
+
.enum(['stdio', 'sse', 'streamable-http'])
|
|
591
|
+
.default(CONFIG_DEFAULTS.discovery.transport),
|
|
451
592
|
/** Remote MCP server URL */
|
|
452
593
|
url: z.string().optional().default(CONFIG_DEFAULTS.discovery.url),
|
|
453
594
|
/** Session ID for remote auth */
|
|
454
595
|
sessionId: z.string().optional().default(CONFIG_DEFAULTS.discovery.sessionId),
|
|
455
|
-
})
|
|
596
|
+
})
|
|
597
|
+
.default(CONFIG_DEFAULTS.discovery);
|
|
456
598
|
/**
|
|
457
599
|
* Registry configuration schema.
|
|
458
600
|
*/
|
|
459
|
-
export const registryConfigSchema = z
|
|
601
|
+
export const registryConfigSchema = z
|
|
602
|
+
.object({
|
|
460
603
|
/** Maximum results to show */
|
|
461
604
|
limit: z.number().int().min(1).max(1000).default(CONFIG_DEFAULTS.registry.limit),
|
|
462
605
|
/** Output as JSON */
|
|
463
606
|
json: z.boolean().default(CONFIG_DEFAULTS.registry.json),
|
|
464
|
-
})
|
|
607
|
+
})
|
|
608
|
+
.default(CONFIG_DEFAULTS.registry);
|
|
465
609
|
/**
|
|
466
610
|
* Golden command configuration schema.
|
|
467
611
|
*/
|
|
468
|
-
export const goldenConfigSchema = z
|
|
612
|
+
export const goldenConfigSchema = z
|
|
613
|
+
.object({
|
|
469
614
|
/** Default JSON args for golden save */
|
|
470
615
|
defaultArgs: z.string().default(CONFIG_DEFAULTS.golden.defaultArgs),
|
|
471
616
|
/** Default comparison mode for golden save */
|
|
472
617
|
mode: z.enum(['exact', 'structural', 'semantic']).default(CONFIG_DEFAULTS.golden.mode),
|
|
473
618
|
/** Output format for compare */
|
|
474
|
-
compareFormat: z
|
|
619
|
+
compareFormat: z
|
|
620
|
+
.enum(['text', 'json', 'markdown'])
|
|
621
|
+
.default(CONFIG_DEFAULTS.golden.compareFormat),
|
|
475
622
|
/** Output format for list */
|
|
476
623
|
listFormat: z.enum(['text', 'json']).default(CONFIG_DEFAULTS.golden.listFormat),
|
|
477
624
|
/** Normalize timestamps by default */
|
|
478
625
|
normalizeTimestamps: z.boolean().default(CONFIG_DEFAULTS.golden.normalizeTimestamps),
|
|
479
626
|
/** Normalize UUIDs by default */
|
|
480
627
|
normalizeUuids: z.boolean().default(CONFIG_DEFAULTS.golden.normalizeUuids),
|
|
481
|
-
})
|
|
628
|
+
})
|
|
629
|
+
.default(CONFIG_DEFAULTS.golden);
|
|
482
630
|
/**
|
|
483
631
|
* Contract command configuration schema.
|
|
484
632
|
*/
|
|
485
|
-
export const contractConfigSchema = z
|
|
633
|
+
export const contractConfigSchema = z
|
|
634
|
+
.object({
|
|
486
635
|
/** Default contract file path */
|
|
487
636
|
path: z.string().optional(),
|
|
488
637
|
/** Validation mode */
|
|
@@ -490,10 +639,16 @@ export const contractConfigSchema = z.object({
|
|
|
490
639
|
/** Output format */
|
|
491
640
|
format: z.enum(['text', 'json', 'markdown']).default(CONFIG_DEFAULTS.contract.format),
|
|
492
641
|
/** Server startup timeout */
|
|
493
|
-
timeout: z
|
|
642
|
+
timeout: z
|
|
643
|
+
.number()
|
|
644
|
+
.int()
|
|
645
|
+
.min(VALIDATION_BOUNDS.TIMEOUT.MIN_MS)
|
|
646
|
+
.max(VALIDATION_BOUNDS.TIMEOUT.MAX_MS)
|
|
647
|
+
.default(CONFIG_DEFAULTS.contract.timeout),
|
|
494
648
|
/** Exit with error when violations are found */
|
|
495
649
|
failOnViolation: z.boolean().default(CONFIG_DEFAULTS.contract.failOnViolation),
|
|
496
|
-
})
|
|
650
|
+
})
|
|
651
|
+
.default(CONFIG_DEFAULTS.contract);
|
|
497
652
|
/**
|
|
498
653
|
* Complete bellwether.yaml configuration schema.
|
|
499
654
|
*
|
package/dist/constants/core.d.ts
CHANGED
|
@@ -484,6 +484,10 @@ export declare const REPORT_SCHEMAS: {
|
|
|
484
484
|
readonly CHECK_REPORT_SCHEMA_FILE: "schemas/bellwether-check.schema.json";
|
|
485
485
|
/** Public schema URL for consumers */
|
|
486
486
|
readonly CHECK_REPORT_SCHEMA_URL: "https://unpkg.com/@dotsetlabs/bellwether/schemas/bellwether-check.schema.json";
|
|
487
|
+
/** Local schema path for bellwether-explore.json */
|
|
488
|
+
readonly EXPLORE_REPORT_SCHEMA_FILE: "schemas/bellwether-explore.schema.json";
|
|
489
|
+
/** Public schema URL for consumers */
|
|
490
|
+
readonly EXPLORE_REPORT_SCHEMA_URL: "https://unpkg.com/@dotsetlabs/bellwether/schemas/bellwether-explore.schema.json";
|
|
487
491
|
};
|
|
488
492
|
/**
|
|
489
493
|
* Regex patterns for validation and parsing.
|