@dotsetlabs/bellwether 1.0.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/README.md +9 -2
- package/dist/baseline/accessors.d.ts +1 -1
- package/dist/baseline/accessors.js +1 -3
- package/dist/baseline/baseline-format.d.ts +287 -0
- package/dist/baseline/baseline-format.js +12 -0
- package/dist/baseline/comparator.js +249 -11
- package/dist/baseline/converter.d.ts +15 -15
- package/dist/baseline/converter.js +46 -34
- package/dist/baseline/diff.d.ts +1 -1
- package/dist/baseline/diff.js +45 -28
- package/dist/baseline/error-analyzer.d.ts +1 -1
- package/dist/baseline/error-analyzer.js +90 -17
- package/dist/baseline/incremental-checker.js +8 -5
- package/dist/baseline/index.d.ts +2 -12
- package/dist/baseline/index.js +3 -23
- package/dist/baseline/performance-tracker.d.ts +0 -1
- package/dist/baseline/performance-tracker.js +13 -20
- package/dist/baseline/response-fingerprint.js +39 -2
- package/dist/baseline/saver.js +41 -10
- package/dist/baseline/schema-compare.d.ts +22 -0
- package/dist/baseline/schema-compare.js +259 -16
- package/dist/baseline/types.d.ts +10 -7
- package/dist/cache/response-cache.d.ts +12 -2
- package/dist/cache/response-cache.js +178 -30
- package/dist/cli/commands/check.js +100 -54
- package/dist/cli/commands/explore.js +34 -14
- package/dist/cli/index.js +13 -3
- package/dist/config/template.js +8 -7
- package/dist/config/validator.d.ts +59 -59
- package/dist/config/validator.js +245 -90
- package/dist/constants/core.d.ts +4 -0
- package/dist/constants/core.js +8 -19
- package/dist/constants/registry.d.ts +17 -0
- package/dist/constants/registry.js +18 -0
- package/dist/constants/testing.d.ts +0 -369
- package/dist/constants/testing.js +18 -456
- package/dist/constants.d.ts +1 -1
- package/dist/constants.js +1 -1
- package/dist/docs/contract.js +131 -83
- package/dist/docs/report.js +8 -5
- package/dist/interview/insights.d.ts +17 -0
- package/dist/interview/insights.js +52 -0
- package/dist/interview/interviewer.js +119 -57
- package/dist/interview/orchestrator.js +49 -22
- package/dist/interview/prompt-test-generator.d.ts +12 -0
- package/dist/interview/prompt-test-generator.js +77 -0
- package/dist/interview/resource-test-generator.d.ts +12 -0
- package/dist/interview/resource-test-generator.js +20 -0
- package/dist/interview/schema-inferrer.js +26 -4
- package/dist/interview/schema-test-generator.js +278 -31
- package/dist/interview/stateful-test-runner.d.ts +3 -0
- package/dist/interview/stateful-test-runner.js +80 -0
- package/dist/interview/types.d.ts +12 -0
- package/dist/llm/anthropic.js +49 -16
- package/dist/llm/client.d.ts +2 -0
- package/dist/llm/client.js +61 -0
- package/dist/llm/ollama.js +9 -4
- package/dist/llm/openai.js +34 -23
- package/dist/transport/base-transport.d.ts +1 -1
- package/dist/transport/http-transport.d.ts +2 -2
- package/dist/transport/http-transport.js +26 -6
- package/dist/transport/mcp-client.d.ts +18 -6
- package/dist/transport/mcp-client.js +50 -20
- package/dist/transport/sse-transport.d.ts +8 -4
- package/dist/transport/sse-transport.js +161 -69
- package/dist/transport/stdio-transport.d.ts +1 -1
- package/dist/transport/stdio-transport.js +1 -1
- package/dist/utils/timeout.d.ts +10 -2
- package/dist/utils/timeout.js +9 -5
- package/dist/version.js +1 -1
- package/dist/workflow/executor.js +18 -13
- package/dist/workflow/loader.js +4 -1
- package/dist/workflow/state-tracker.js +22 -18
- package/man/bellwether.1 +204 -0
- package/man/bellwether.1.md +148 -0
- package/package.json +6 -7
- package/schemas/bellwether-check.schema.json +185 -0
- package/schemas/bellwether-explore.schema.json +837 -0
- package/scripts/completions/bellwether.bash +10 -4
- package/scripts/completions/bellwether.zsh +55 -2
|
@@ -13,22 +13,23 @@ import { MCPClient } from '../../transport/mcp-client.js';
|
|
|
13
13
|
import { discover } from '../../discovery/discovery.js';
|
|
14
14
|
import { Interviewer } from '../../interview/interviewer.js';
|
|
15
15
|
import { generateAgentsMd, generateJsonReport } from '../../docs/generator.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError, parseCommandString } from '../../config/loader.js';
|
|
16
|
+
import { loadConfig, ConfigNotFoundError, parseCommandString, } from '../../config/loader.js';
|
|
17
17
|
import { validateConfigForExplore } from '../../config/validator.js';
|
|
18
18
|
import { CostTracker, estimateInterviewCost, estimateInterviewTime, formatCostAndTimeEstimate, suggestOptimizations, formatOptimizationSuggestions, } from '../../cost/index.js';
|
|
19
19
|
import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
|
|
20
|
-
import { EXIT_CODES, WORKFLOW, PATHS } from '../../constants.js';
|
|
20
|
+
import { EXIT_CODES, WORKFLOW, PATHS, REPORT_SCHEMAS } from '../../constants.js';
|
|
21
21
|
import { FallbackLLMClient } from '../../llm/fallback.js';
|
|
22
22
|
import { getGlobalCache, resetGlobalCache } from '../../cache/response-cache.js';
|
|
23
23
|
import { InterviewProgressBar, formatExploreBanner } from '../utils/progress.js';
|
|
24
24
|
import { parsePersonas } from '../../persona/builtins.js';
|
|
25
|
-
import { loadScenariosFromFile, tryLoadDefaultScenarios, DEFAULT_SCENARIOS_FILE } from '../../scenarios/index.js';
|
|
26
|
-
import { loadWorkflowsFromFile, tryLoadDefaultWorkflows, DEFAULT_WORKFLOWS_FILE } from '../../workflow/loader.js';
|
|
25
|
+
import { loadScenariosFromFile, tryLoadDefaultScenarios, DEFAULT_SCENARIOS_FILE, } from '../../scenarios/index.js';
|
|
26
|
+
import { loadWorkflowsFromFile, tryLoadDefaultWorkflows, DEFAULT_WORKFLOWS_FILE, } from '../../workflow/loader.js';
|
|
27
27
|
import * as output from '../output.js';
|
|
28
28
|
import { StreamingDisplay } from '../output.js';
|
|
29
|
-
import { suppressLogs, restoreLogLevel, configureLogger } from '../../logging/logger.js';
|
|
29
|
+
import { suppressLogs, restoreLogLevel, configureLogger, } from '../../logging/logger.js';
|
|
30
30
|
import { extractServerContextFromArgs } from '../utils/server-context.js';
|
|
31
31
|
import { isCI } from '../utils/env.js';
|
|
32
|
+
import { buildInterviewInsights } from '../../interview/insights.js';
|
|
32
33
|
/**
|
|
33
34
|
* Wrapper to parse personas with warning output.
|
|
34
35
|
*/
|
|
@@ -120,7 +121,7 @@ export const exploreCommand = new Command('explore')
|
|
|
120
121
|
metricsCollector.startInterview();
|
|
121
122
|
// Initialize cache
|
|
122
123
|
resetGlobalCache();
|
|
123
|
-
const cache = getGlobalCache({ enabled: cacheEnabled });
|
|
124
|
+
const cache = getGlobalCache({ enabled: cacheEnabled, dir: config.cache.dir });
|
|
124
125
|
if (cacheEnabled && verbose) {
|
|
125
126
|
output.info('Response caching enabled');
|
|
126
127
|
}
|
|
@@ -138,7 +139,13 @@ export const exploreCommand = new Command('explore')
|
|
|
138
139
|
};
|
|
139
140
|
try {
|
|
140
141
|
llmClient = new FallbackLLMClient({
|
|
141
|
-
providers: [
|
|
142
|
+
providers: [
|
|
143
|
+
{
|
|
144
|
+
provider,
|
|
145
|
+
model,
|
|
146
|
+
baseUrl: provider === 'ollama' ? config.llm.ollama.baseUrl : undefined,
|
|
147
|
+
},
|
|
148
|
+
],
|
|
142
149
|
useOllamaFallback: true,
|
|
143
150
|
onUsage: onUsageCallback,
|
|
144
151
|
});
|
|
@@ -166,9 +173,12 @@ export const exploreCommand = new Command('explore')
|
|
|
166
173
|
}
|
|
167
174
|
// Discovery phase
|
|
168
175
|
output.info('Discovering capabilities...');
|
|
169
|
-
const discovery = await discover(mcpClient, transport === 'stdio' ? serverCommand : remoteUrl ?? serverCommand, transport === 'stdio' ? args : []);
|
|
176
|
+
const discovery = await discover(mcpClient, transport === 'stdio' ? serverCommand : (remoteUrl ?? serverCommand), transport === 'stdio' ? args : []);
|
|
170
177
|
const resourceCount = discovery.resources?.length ?? 0;
|
|
171
|
-
const discoveryParts = [
|
|
178
|
+
const discoveryParts = [
|
|
179
|
+
`${discovery.tools.length} tools`,
|
|
180
|
+
`${discovery.prompts.length} prompts`,
|
|
181
|
+
];
|
|
172
182
|
if (resourceCount > 0) {
|
|
173
183
|
discoveryParts.push(`${resourceCount} resources`);
|
|
174
184
|
}
|
|
@@ -277,13 +287,17 @@ export const exploreCommand = new Command('explore')
|
|
|
277
287
|
let prefix = '';
|
|
278
288
|
switch (opType) {
|
|
279
289
|
case 'generate-questions':
|
|
280
|
-
prefix = context
|
|
290
|
+
prefix = context
|
|
291
|
+
? `\n Generating questions for ${context}... `
|
|
292
|
+
: '\n Generating questions... ';
|
|
281
293
|
break;
|
|
282
294
|
case 'analyze':
|
|
283
295
|
prefix = context ? `\n Analyzing ${context}... ` : '\n Analyzing... ';
|
|
284
296
|
break;
|
|
285
297
|
case 'synthesize-tool':
|
|
286
|
-
prefix = context
|
|
298
|
+
prefix = context
|
|
299
|
+
? `\n Synthesizing profile for ${context}... `
|
|
300
|
+
: '\n Synthesizing profile... ';
|
|
287
301
|
break;
|
|
288
302
|
case 'synthesize-overall':
|
|
289
303
|
prefix = '\n Synthesizing overall findings... ';
|
|
@@ -365,6 +379,8 @@ export const exploreCommand = new Command('explore')
|
|
|
365
379
|
};
|
|
366
380
|
output.info('Starting exploration...\n');
|
|
367
381
|
const result = await interviewer.interview(mcpClient, discovery, progressCallback);
|
|
382
|
+
const insights = buildInterviewInsights(result);
|
|
383
|
+
const enrichedResult = { ...result, ...insights };
|
|
368
384
|
progressBar.stop();
|
|
369
385
|
if (!verbose) {
|
|
370
386
|
output.newline();
|
|
@@ -375,16 +391,20 @@ export const exploreCommand = new Command('explore')
|
|
|
375
391
|
if (docsDir !== outputDir) {
|
|
376
392
|
mkdirSync(docsDir, { recursive: true });
|
|
377
393
|
}
|
|
378
|
-
const writeDocs = outputFormat === 'both' || outputFormat === '
|
|
394
|
+
const writeDocs = outputFormat === 'both' || outputFormat === 'docs';
|
|
379
395
|
const writeJson = outputFormat === 'both' || outputFormat === 'json';
|
|
380
396
|
if (writeDocs) {
|
|
381
|
-
const agentsMd = generateAgentsMd(
|
|
397
|
+
const agentsMd = generateAgentsMd(enrichedResult);
|
|
382
398
|
const agentsMdPath = join(docsDir, config.output.files.agentsDoc);
|
|
383
399
|
writeFileSync(agentsMdPath, agentsMd);
|
|
384
400
|
output.info(`Written: ${agentsMdPath}`);
|
|
385
401
|
}
|
|
386
402
|
if (writeJson) {
|
|
387
|
-
const jsonReport = generateJsonReport(
|
|
403
|
+
const jsonReport = generateJsonReport(enrichedResult, {
|
|
404
|
+
schemaUrl: REPORT_SCHEMAS.EXPLORE_REPORT_SCHEMA_URL,
|
|
405
|
+
schemaPath: REPORT_SCHEMAS.EXPLORE_REPORT_SCHEMA_FILE,
|
|
406
|
+
validate: true,
|
|
407
|
+
});
|
|
388
408
|
const jsonPath = join(outputDir, config.output.files.exploreReport);
|
|
389
409
|
writeFileSync(jsonPath, jsonReport);
|
|
390
410
|
output.info(`Written: ${jsonPath}`);
|
package/dist/cli/index.js
CHANGED
|
@@ -16,7 +16,7 @@ if (existsSync(globalEnvPath)) {
|
|
|
16
16
|
config({ path: globalEnvPath, quiet: true });
|
|
17
17
|
}
|
|
18
18
|
// Then load project .env (overrides global settings)
|
|
19
|
-
config({ quiet: true });
|
|
19
|
+
config({ quiet: true, override: true });
|
|
20
20
|
function normalizeEncryptedEnvVar(key) {
|
|
21
21
|
const value = process.env[key];
|
|
22
22
|
if (!value || !isEncryptedEnvValue(value)) {
|
|
@@ -115,6 +115,14 @@ Check MCP servers for drift. Explore behavior. Generate documentation.
|
|
|
115
115
|
Commands:
|
|
116
116
|
check - Schema validation and drift detection (free, fast, deterministic)
|
|
117
117
|
explore - LLM-powered behavioral exploration and documentation
|
|
118
|
+
discover - Quick capability discovery (no tests)
|
|
119
|
+
registry - Search the MCP Registry
|
|
120
|
+
baseline - Manage baselines (save/compare/accept/diff/show)
|
|
121
|
+
golden - Golden output regression testing
|
|
122
|
+
contract - Contract validation (generate/validate/show)
|
|
123
|
+
watch - Continuous checking on file changes
|
|
124
|
+
auth - Manage LLM provider API keys
|
|
125
|
+
validate-config - Validate bellwether.yaml without running tests
|
|
118
126
|
|
|
119
127
|
For more information on a specific command, use:
|
|
120
128
|
bellwether <command> --help`)
|
|
@@ -167,9 +175,11 @@ program.configureHelp({
|
|
|
167
175
|
subcommandTerm: (cmd) => `${cmd.name()} ${cmd.usage()}`,
|
|
168
176
|
});
|
|
169
177
|
// Load keychain credentials, then parse commands
|
|
170
|
-
loadKeychainCredentials()
|
|
178
|
+
loadKeychainCredentials()
|
|
179
|
+
.then(() => {
|
|
171
180
|
program.parse();
|
|
172
|
-
})
|
|
181
|
+
})
|
|
182
|
+
.catch(() => {
|
|
173
183
|
// If keychain loading fails, still parse commands
|
|
174
184
|
program.parse();
|
|
175
185
|
});
|
package/dist/config/template.js
CHANGED
|
@@ -23,13 +23,11 @@ export function generateConfigTemplate(options = {}) {
|
|
|
23
23
|
// Override security.enabled if preset specifies it
|
|
24
24
|
const securityEnabledValue = securityEnabled ? 'true' : String(defaults.check.security.enabled);
|
|
25
25
|
const serverArgsYaml = serverArgs.length > 0
|
|
26
|
-
? `\n args:\n${serverArgs.map(arg => ` - "${arg}"`).join('\n')}`
|
|
26
|
+
? `\n args:\n${serverArgs.map((arg) => ` - "${arg}"`).join('\n')}`
|
|
27
27
|
: '\n args: []';
|
|
28
28
|
const presetComment = preset ? `# Generated with: bellwether init --preset ${preset}\n` : '';
|
|
29
29
|
// Generate env section if env vars were detected
|
|
30
|
-
const envVarsYaml = envVars.length > 0
|
|
31
|
-
? `\n env:\n${envVars.map(v => ` ${v}: "\${${v}}"`).join('\n')}`
|
|
32
|
-
: '';
|
|
30
|
+
const envVarsYaml = envVars.length > 0 ? `\n env:\n${envVars.map((v) => ` ${v}: "\${${v}}"`).join('\n')}` : '';
|
|
33
31
|
return `# Bellwether Configuration
|
|
34
32
|
# Generated by: bellwether init
|
|
35
33
|
# Docs: https://docs.bellwether.sh/guides/configuration
|
|
@@ -61,10 +59,12 @@ server:
|
|
|
61
59
|
timeout: ${defaults.server.timeout}
|
|
62
60
|
|
|
63
61
|
# Additional environment variables for the server process
|
|
64
|
-
# Use \${VAR} syntax to reference environment variables${envVarsYaml}${envVars.length === 0
|
|
62
|
+
# Use \${VAR} syntax to reference environment variables${envVarsYaml}${envVars.length === 0
|
|
63
|
+
? `
|
|
65
64
|
# env:
|
|
66
65
|
# NODE_ENV: production
|
|
67
|
-
# API_KEY: "\${API_KEY}"`
|
|
66
|
+
# API_KEY: "\${API_KEY}"`
|
|
67
|
+
: ''}
|
|
68
68
|
|
|
69
69
|
# =============================================================================
|
|
70
70
|
# SCENARIOS (used by both commands)
|
|
@@ -91,7 +91,8 @@ output:
|
|
|
91
91
|
# These are kept in root by default for visibility
|
|
92
92
|
docsDir: "${defaults.output.docsDir}"
|
|
93
93
|
|
|
94
|
-
# Output format:
|
|
94
|
+
# Output format: docs (markdown only), json (JSON only), or both
|
|
95
|
+
# Legacy alias: agents.md (treated as docs)
|
|
95
96
|
format: ${defaults.output.format}
|
|
96
97
|
|
|
97
98
|
# Example output settings (for CONTRACT.md and AGENTS.md)
|
|
@@ -247,7 +247,7 @@ export declare const outputConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
247
247
|
/** Output directory for documentation files (CONTRACT.md, AGENTS.md) */
|
|
248
248
|
docsDir: z.ZodDefault<z.ZodString>;
|
|
249
249
|
/** Output format */
|
|
250
|
-
format: z.ZodDefault<z.ZodEnum<["
|
|
250
|
+
format: z.ZodDefault<z.ZodEffects<z.ZodEnum<["docs", "json", "both"]>, "json" | "both" | "docs", unknown>>;
|
|
251
251
|
/** Example output settings */
|
|
252
252
|
examples: z.ZodDefault<z.ZodObject<{
|
|
253
253
|
/** Include full (non-truncated) examples */
|
|
@@ -287,7 +287,7 @@ export declare const outputConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
287
287
|
agentsDoc?: string | undefined;
|
|
288
288
|
}>>;
|
|
289
289
|
}, "strip", z.ZodTypeAny, {
|
|
290
|
-
format: "json" | "both" | "
|
|
290
|
+
format: "json" | "both" | "docs";
|
|
291
291
|
examples: {
|
|
292
292
|
maxLength: number;
|
|
293
293
|
full: boolean;
|
|
@@ -302,7 +302,7 @@ export declare const outputConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
302
302
|
};
|
|
303
303
|
docsDir: string;
|
|
304
304
|
}, {
|
|
305
|
-
format?:
|
|
305
|
+
format?: unknown;
|
|
306
306
|
examples?: {
|
|
307
307
|
maxLength?: number | undefined;
|
|
308
308
|
full?: boolean | undefined;
|
|
@@ -329,17 +329,17 @@ export declare const severityConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
329
329
|
/** Suppress warning-level changes from output */
|
|
330
330
|
suppressWarnings: z.ZodDefault<z.ZodBoolean>;
|
|
331
331
|
/** Custom severity overrides per aspect */
|
|
332
|
-
aspectOverrides: z.ZodOptional<z.ZodRecord<z.ZodEnum<["response_format", "response_structure", "error_handling", "error_pattern", "security", "performance", "schema", "description"]>, z.ZodEnum<["none", "info", "warning", "breaking"]>>>;
|
|
332
|
+
aspectOverrides: z.ZodOptional<z.ZodRecord<z.ZodEnum<["response_format", "response_structure", "response_schema_evolution", "error_handling", "error_pattern", "security", "performance", "schema", "description", "prompt", "resource", "server", "capability"]>, z.ZodEnum<["none", "info", "warning", "breaking"]>>>;
|
|
333
333
|
}, "strip", z.ZodTypeAny, {
|
|
334
334
|
minimumSeverity: "none" | "info" | "warning" | "breaking";
|
|
335
335
|
failOnSeverity: "none" | "info" | "warning" | "breaking";
|
|
336
336
|
suppressWarnings: boolean;
|
|
337
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
337
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
338
338
|
}, {
|
|
339
339
|
minimumSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
340
340
|
failOnSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
341
341
|
suppressWarnings?: boolean | undefined;
|
|
342
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
342
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
343
343
|
}>>;
|
|
344
344
|
/**
|
|
345
345
|
* Security testing configuration schema.
|
|
@@ -864,17 +864,17 @@ export declare const baselineConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
864
864
|
/** Suppress warning-level changes from output */
|
|
865
865
|
suppressWarnings: z.ZodDefault<z.ZodBoolean>;
|
|
866
866
|
/** Custom severity overrides per aspect */
|
|
867
|
-
aspectOverrides: z.ZodOptional<z.ZodRecord<z.ZodEnum<["response_format", "response_structure", "error_handling", "error_pattern", "security", "performance", "schema", "description"]>, z.ZodEnum<["none", "info", "warning", "breaking"]>>>;
|
|
867
|
+
aspectOverrides: z.ZodOptional<z.ZodRecord<z.ZodEnum<["response_format", "response_structure", "response_schema_evolution", "error_handling", "error_pattern", "security", "performance", "schema", "description", "prompt", "resource", "server", "capability"]>, z.ZodEnum<["none", "info", "warning", "breaking"]>>>;
|
|
868
868
|
}, "strip", z.ZodTypeAny, {
|
|
869
869
|
minimumSeverity: "none" | "info" | "warning" | "breaking";
|
|
870
870
|
failOnSeverity: "none" | "info" | "warning" | "breaking";
|
|
871
871
|
suppressWarnings: boolean;
|
|
872
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
872
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
873
873
|
}, {
|
|
874
874
|
minimumSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
875
875
|
failOnSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
876
876
|
suppressWarnings?: boolean | undefined;
|
|
877
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
877
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
878
878
|
}>>;
|
|
879
879
|
}, "strip", z.ZodTypeAny, {
|
|
880
880
|
path: string;
|
|
@@ -882,7 +882,7 @@ export declare const baselineConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
882
882
|
minimumSeverity: "none" | "info" | "warning" | "breaking";
|
|
883
883
|
failOnSeverity: "none" | "info" | "warning" | "breaking";
|
|
884
884
|
suppressWarnings: boolean;
|
|
885
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
885
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
886
886
|
};
|
|
887
887
|
failOnDrift: boolean;
|
|
888
888
|
outputFormat: "text" | "json" | "markdown" | "compact";
|
|
@@ -894,7 +894,7 @@ export declare const baselineConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
894
894
|
minimumSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
895
895
|
failOnSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
896
896
|
suppressWarnings?: boolean | undefined;
|
|
897
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
897
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
898
898
|
} | undefined;
|
|
899
899
|
savePath?: string | undefined;
|
|
900
900
|
comparePath?: string | undefined;
|
|
@@ -1013,14 +1013,14 @@ export declare const goldenConfigSchema: z.ZodDefault<z.ZodObject<{
|
|
|
1013
1013
|
/** Normalize UUIDs by default */
|
|
1014
1014
|
normalizeUuids: z.ZodDefault<z.ZodBoolean>;
|
|
1015
1015
|
}, "strip", z.ZodTypeAny, {
|
|
1016
|
-
mode: "
|
|
1016
|
+
mode: "structural" | "exact" | "semantic";
|
|
1017
1017
|
defaultArgs: string;
|
|
1018
1018
|
compareFormat: "text" | "json" | "markdown";
|
|
1019
1019
|
listFormat: "text" | "json";
|
|
1020
1020
|
normalizeTimestamps: boolean;
|
|
1021
1021
|
normalizeUuids: boolean;
|
|
1022
1022
|
}, {
|
|
1023
|
-
mode?: "
|
|
1023
|
+
mode?: "structural" | "exact" | "semantic" | undefined;
|
|
1024
1024
|
defaultArgs?: string | undefined;
|
|
1025
1025
|
compareFormat?: "text" | "json" | "markdown" | undefined;
|
|
1026
1026
|
listFormat?: "text" | "json" | undefined;
|
|
@@ -1525,7 +1525,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1525
1525
|
/** Output directory for documentation files (CONTRACT.md, AGENTS.md) */
|
|
1526
1526
|
docsDir: z.ZodDefault<z.ZodString>;
|
|
1527
1527
|
/** Output format */
|
|
1528
|
-
format: z.ZodDefault<z.ZodEnum<["
|
|
1528
|
+
format: z.ZodDefault<z.ZodEffects<z.ZodEnum<["docs", "json", "both"]>, "json" | "both" | "docs", unknown>>;
|
|
1529
1529
|
/** Example output settings */
|
|
1530
1530
|
examples: z.ZodDefault<z.ZodObject<{
|
|
1531
1531
|
/** Include full (non-truncated) examples */
|
|
@@ -1565,7 +1565,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1565
1565
|
agentsDoc?: string | undefined;
|
|
1566
1566
|
}>>;
|
|
1567
1567
|
}, "strip", z.ZodTypeAny, {
|
|
1568
|
-
format: "json" | "both" | "
|
|
1568
|
+
format: "json" | "both" | "docs";
|
|
1569
1569
|
examples: {
|
|
1570
1570
|
maxLength: number;
|
|
1571
1571
|
full: boolean;
|
|
@@ -1580,7 +1580,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1580
1580
|
};
|
|
1581
1581
|
docsDir: string;
|
|
1582
1582
|
}, {
|
|
1583
|
-
format?:
|
|
1583
|
+
format?: unknown;
|
|
1584
1584
|
examples?: {
|
|
1585
1585
|
maxLength?: number | undefined;
|
|
1586
1586
|
full?: boolean | undefined;
|
|
@@ -1616,17 +1616,17 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1616
1616
|
/** Suppress warning-level changes from output */
|
|
1617
1617
|
suppressWarnings: z.ZodDefault<z.ZodBoolean>;
|
|
1618
1618
|
/** Custom severity overrides per aspect */
|
|
1619
|
-
aspectOverrides: z.ZodOptional<z.ZodRecord<z.ZodEnum<["response_format", "response_structure", "error_handling", "error_pattern", "security", "performance", "schema", "description"]>, z.ZodEnum<["none", "info", "warning", "breaking"]>>>;
|
|
1619
|
+
aspectOverrides: z.ZodOptional<z.ZodRecord<z.ZodEnum<["response_format", "response_structure", "response_schema_evolution", "error_handling", "error_pattern", "security", "performance", "schema", "description", "prompt", "resource", "server", "capability"]>, z.ZodEnum<["none", "info", "warning", "breaking"]>>>;
|
|
1620
1620
|
}, "strip", z.ZodTypeAny, {
|
|
1621
1621
|
minimumSeverity: "none" | "info" | "warning" | "breaking";
|
|
1622
1622
|
failOnSeverity: "none" | "info" | "warning" | "breaking";
|
|
1623
1623
|
suppressWarnings: boolean;
|
|
1624
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1624
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1625
1625
|
}, {
|
|
1626
1626
|
minimumSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
1627
1627
|
failOnSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
1628
1628
|
suppressWarnings?: boolean | undefined;
|
|
1629
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1629
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1630
1630
|
}>>;
|
|
1631
1631
|
}, "strip", z.ZodTypeAny, {
|
|
1632
1632
|
path: string;
|
|
@@ -1634,7 +1634,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1634
1634
|
minimumSeverity: "none" | "info" | "warning" | "breaking";
|
|
1635
1635
|
failOnSeverity: "none" | "info" | "warning" | "breaking";
|
|
1636
1636
|
suppressWarnings: boolean;
|
|
1637
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1637
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1638
1638
|
};
|
|
1639
1639
|
failOnDrift: boolean;
|
|
1640
1640
|
outputFormat: "text" | "json" | "markdown" | "compact";
|
|
@@ -1646,7 +1646,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1646
1646
|
minimumSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
1647
1647
|
failOnSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
1648
1648
|
suppressWarnings?: boolean | undefined;
|
|
1649
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1649
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1650
1650
|
} | undefined;
|
|
1651
1651
|
savePath?: string | undefined;
|
|
1652
1652
|
comparePath?: string | undefined;
|
|
@@ -1753,14 +1753,14 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1753
1753
|
/** Normalize UUIDs by default */
|
|
1754
1754
|
normalizeUuids: z.ZodDefault<z.ZodBoolean>;
|
|
1755
1755
|
}, "strip", z.ZodTypeAny, {
|
|
1756
|
-
mode: "
|
|
1756
|
+
mode: "structural" | "exact" | "semantic";
|
|
1757
1757
|
defaultArgs: string;
|
|
1758
1758
|
compareFormat: "text" | "json" | "markdown";
|
|
1759
1759
|
listFormat: "text" | "json";
|
|
1760
1760
|
normalizeTimestamps: boolean;
|
|
1761
1761
|
normalizeUuids: boolean;
|
|
1762
1762
|
}, {
|
|
1763
|
-
mode?: "
|
|
1763
|
+
mode?: "structural" | "exact" | "semantic" | undefined;
|
|
1764
1764
|
defaultArgs?: string | undefined;
|
|
1765
1765
|
compareFormat?: "text" | "json" | "markdown" | undefined;
|
|
1766
1766
|
listFormat?: "text" | "json" | undefined;
|
|
@@ -1793,22 +1793,6 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1793
1793
|
failOnViolation?: boolean | undefined;
|
|
1794
1794
|
}>>;
|
|
1795
1795
|
}, "strip", z.ZodTypeAny, {
|
|
1796
|
-
output: {
|
|
1797
|
-
format: "json" | "both" | "agents.md";
|
|
1798
|
-
examples: {
|
|
1799
|
-
maxLength: number;
|
|
1800
|
-
full: boolean;
|
|
1801
|
-
maxPerTool: number;
|
|
1802
|
-
};
|
|
1803
|
-
dir: string;
|
|
1804
|
-
files: {
|
|
1805
|
-
checkReport: string;
|
|
1806
|
-
exploreReport: string;
|
|
1807
|
-
contractDoc: string;
|
|
1808
|
-
agentsDoc: string;
|
|
1809
|
-
};
|
|
1810
|
-
docsDir: string;
|
|
1811
|
-
};
|
|
1812
1796
|
server: {
|
|
1813
1797
|
command: string;
|
|
1814
1798
|
args: string[];
|
|
@@ -1886,6 +1870,22 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1886
1870
|
parallelPersonas: boolean;
|
|
1887
1871
|
personaConcurrency: number;
|
|
1888
1872
|
};
|
|
1873
|
+
output: {
|
|
1874
|
+
format: "json" | "both" | "docs";
|
|
1875
|
+
examples: {
|
|
1876
|
+
maxLength: number;
|
|
1877
|
+
full: boolean;
|
|
1878
|
+
maxPerTool: number;
|
|
1879
|
+
};
|
|
1880
|
+
dir: string;
|
|
1881
|
+
files: {
|
|
1882
|
+
checkReport: string;
|
|
1883
|
+
exploreReport: string;
|
|
1884
|
+
contractDoc: string;
|
|
1885
|
+
agentsDoc: string;
|
|
1886
|
+
};
|
|
1887
|
+
docsDir: string;
|
|
1888
|
+
};
|
|
1889
1889
|
cache: {
|
|
1890
1890
|
enabled: boolean;
|
|
1891
1891
|
dir: string;
|
|
@@ -1924,7 +1924,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1924
1924
|
minimumSeverity: "none" | "info" | "warning" | "breaking";
|
|
1925
1925
|
failOnSeverity: "none" | "info" | "warning" | "breaking";
|
|
1926
1926
|
suppressWarnings: boolean;
|
|
1927
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1927
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
1928
1928
|
};
|
|
1929
1929
|
failOnDrift: boolean;
|
|
1930
1930
|
outputFormat: "text" | "json" | "markdown" | "compact";
|
|
@@ -1946,7 +1946,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1946
1946
|
limit: number;
|
|
1947
1947
|
};
|
|
1948
1948
|
golden: {
|
|
1949
|
-
mode: "
|
|
1949
|
+
mode: "structural" | "exact" | "semantic";
|
|
1950
1950
|
defaultArgs: string;
|
|
1951
1951
|
compareFormat: "text" | "json" | "markdown";
|
|
1952
1952
|
listFormat: "text" | "json";
|
|
@@ -1961,22 +1961,6 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
1961
1961
|
path?: string | undefined;
|
|
1962
1962
|
};
|
|
1963
1963
|
}, {
|
|
1964
|
-
output?: {
|
|
1965
|
-
format?: "json" | "both" | "agents.md" | undefined;
|
|
1966
|
-
examples?: {
|
|
1967
|
-
maxLength?: number | undefined;
|
|
1968
|
-
full?: boolean | undefined;
|
|
1969
|
-
maxPerTool?: number | undefined;
|
|
1970
|
-
} | undefined;
|
|
1971
|
-
dir?: string | undefined;
|
|
1972
|
-
files?: {
|
|
1973
|
-
checkReport?: string | undefined;
|
|
1974
|
-
exploreReport?: string | undefined;
|
|
1975
|
-
contractDoc?: string | undefined;
|
|
1976
|
-
agentsDoc?: string | undefined;
|
|
1977
|
-
} | undefined;
|
|
1978
|
-
docsDir?: string | undefined;
|
|
1979
|
-
} | undefined;
|
|
1980
1964
|
server?: {
|
|
1981
1965
|
command?: string | undefined;
|
|
1982
1966
|
args?: string[] | undefined;
|
|
@@ -2054,6 +2038,22 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
2054
2038
|
parallelPersonas?: boolean | undefined;
|
|
2055
2039
|
personaConcurrency?: number | undefined;
|
|
2056
2040
|
} | undefined;
|
|
2041
|
+
output?: {
|
|
2042
|
+
format?: unknown;
|
|
2043
|
+
examples?: {
|
|
2044
|
+
maxLength?: number | undefined;
|
|
2045
|
+
full?: boolean | undefined;
|
|
2046
|
+
maxPerTool?: number | undefined;
|
|
2047
|
+
} | undefined;
|
|
2048
|
+
dir?: string | undefined;
|
|
2049
|
+
files?: {
|
|
2050
|
+
checkReport?: string | undefined;
|
|
2051
|
+
exploreReport?: string | undefined;
|
|
2052
|
+
contractDoc?: string | undefined;
|
|
2053
|
+
agentsDoc?: string | undefined;
|
|
2054
|
+
} | undefined;
|
|
2055
|
+
docsDir?: string | undefined;
|
|
2056
|
+
} | undefined;
|
|
2057
2057
|
cache?: {
|
|
2058
2058
|
enabled?: boolean | undefined;
|
|
2059
2059
|
dir?: string | undefined;
|
|
@@ -2092,7 +2092,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
2092
2092
|
minimumSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
2093
2093
|
failOnSeverity?: "none" | "info" | "warning" | "breaking" | undefined;
|
|
2094
2094
|
suppressWarnings?: boolean | undefined;
|
|
2095
|
-
aspectOverrides?: Partial<Record<"error_handling" | "security" | "response_format" | "response_structure" | "error_pattern" | "performance" | "schema" | "description", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
2095
|
+
aspectOverrides?: Partial<Record<"error_handling" | "resource" | "server" | "security" | "response_format" | "response_structure" | "response_schema_evolution" | "error_pattern" | "performance" | "schema" | "description" | "prompt" | "capability", "none" | "info" | "warning" | "breaking">> | undefined;
|
|
2096
2096
|
} | undefined;
|
|
2097
2097
|
savePath?: string | undefined;
|
|
2098
2098
|
comparePath?: string | undefined;
|
|
@@ -2114,7 +2114,7 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
|
|
|
2114
2114
|
limit?: number | undefined;
|
|
2115
2115
|
} | undefined;
|
|
2116
2116
|
golden?: {
|
|
2117
|
-
mode?: "
|
|
2117
|
+
mode?: "structural" | "exact" | "semantic" | undefined;
|
|
2118
2118
|
defaultArgs?: string | undefined;
|
|
2119
2119
|
compareFormat?: "text" | "json" | "markdown" | undefined;
|
|
2120
2120
|
listFormat?: "text" | "json" | undefined;
|