@dotsetlabs/bellwether 0.10.0

package/dist/cli/commands/preset.d.ts

@@ -0,0 +1,33 @@
+import { Command } from 'commander';
+/**
+ * Preset configuration structure.
+ */
+interface Preset {
+    name: string;
+    llm: {
+        provider: 'openai' | 'anthropic' | 'ollama';
+        model?: string;
+    };
+    interview: {
+        maxQuestionsPerTool?: number;
+        personas?: string[];
+        timeout?: number;
+        skipErrorTests?: boolean;
+    };
+    output: {
+        format?: 'markdown' | 'json' | 'both';
+        outputDir?: string;
+    };
+}
+export declare const presetCommand: Command;
+/**
+ * Get the preset config to merge with main config.
+ * Returns null if no current preset is set.
+ */
+export declare function getActivePresetConfig(): Partial<{
+    llm: Preset['llm'];
+    interview: Preset['interview'];
+    output: Preset['output'];
+}> | null;
+export {};
+//# sourceMappingURL=preset.d.ts.map

package/dist/cli/commands/preset.js

@@ -0,0 +1,297 @@
+import { Command } from 'commander';
+import { existsSync, mkdirSync, writeFileSync, readFileSync, readdirSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+import { stringify } from 'yaml';
+import { z } from 'zod';
+import { parseYamlSecure } from '../../utils/yaml-parser.js';
+import * as output from '../output.js';
+/**
+ * Zod schema for preset validation to prevent untrusted YAML injection.
+ */
+const presetSchema = z.object({
+    name: z.string().min(1).max(64),
+    llm: z.object({
+        provider: z.enum(['openai', 'anthropic', 'ollama']),
+        model: z.string().optional(),
+    }),
+    interview: z.object({
+        maxQuestionsPerTool: z.number().positive().optional(),
+        personas: z.array(z.string()).optional(),
+        timeout: z.number().positive().optional(),
+        skipErrorTests: z.boolean().optional(),
+    }).optional().default({}),
+    output: z.object({
+        format: z.enum(['markdown', 'json', 'both']).optional(),
+        outputDir: z.string().optional(),
+    }).optional().default({}),
+});
+const PROFILES_DIR = join(homedir(), '.bellwether', 'presets');
+const CURRENT_PROFILE_FILE = join(homedir(), '.bellwether', 'current-preset');
+/**
+ * Validate preset name to prevent path traversal attacks.
+ * Only alphanumeric characters, hyphens, and underscores are allowed.
+ */
+function isValidPresetName(name) {
+    return /^[a-zA-Z0-9_-]+$/.test(name) && name.length > 0 && name.length <= 64;
+}
+function ensurePresetsDir() {
+    if (!existsSync(PROFILES_DIR)) {
+        mkdirSync(PROFILES_DIR, { recursive: true });
+    }
+}
+function getPresetPath(name) {
+    if (!isValidPresetName(name)) {
+        throw new Error(`Invalid preset name: "${name}". Only alphanumeric characters, hyphens, and underscores are allowed.`);
+    }
+    return join(PROFILES_DIR, `${name}.yaml`);
+}
+function loadPreset(name) {
+    const path = getPresetPath(name);
+    if (!existsSync(path)) {
+        return null;
+    }
+    const content = readFileSync(path, 'utf-8');
+    return parseYamlSecure(content);
+}
+function savePreset(preset) {
+    ensurePresetsDir();
+    const path = getPresetPath(preset.name);
+    writeFileSync(path, stringify(preset));
+}
+function deletePreset(name) {
+    const path = getPresetPath(name);
+    if (!existsSync(path)) {
+        return false;
+    }
+    unlinkSync(path);
+    return true;
+}
+function listPresets() {
+    ensurePresetsDir();
+    return readdirSync(PROFILES_DIR)
+        .filter(f => f.endsWith('.yaml'))
+        .map(f => f.replace('.yaml', ''));
+}
+function getCurrentPreset() {
+    if (!existsSync(CURRENT_PROFILE_FILE)) {
+        return null;
+    }
+    return readFileSync(CURRENT_PROFILE_FILE, 'utf-8').trim();
+}
+function setCurrentPreset(name) {
+    ensurePresetsDir();
+    if (name) {
+        writeFileSync(CURRENT_PROFILE_FILE, name);
+    }
+    else if (existsSync(CURRENT_PROFILE_FILE)) {
+        unlinkSync(CURRENT_PROFILE_FILE);
+    }
+}
+export const presetCommand = new Command('preset')
+    .description('Manage test configuration presets (deprecated: use bellwether init --preset)')
+    .addHelpText('after', `
+Note: Runtime presets are deprecated.
+Use 'bellwether init --preset <name>' to create a config file with preset settings.
+
+Available init presets: ci, security, thorough, local
+`)
+    .addCommand(new Command('create')
+    .description('Create a new preset (deprecated)')
+    .argument('<name>', 'Preset name')
+    .option('-p, --provider <provider>', 'LLM provider (openai, anthropic, ollama)', 'openai')
+    .option('-m, --model <model>', 'LLM model')
+    .option('-q, --max-questions <n>', 'Max questions per tool')
+    .option('--personas <list>', 'Comma-separated list of personas')
+    .option('-f, --format <format>', 'Output format (markdown, json, both)')
+    .option('--use', 'Set as current preset after creation')
+    .action((name, options) => {
+    output.warn('Warning: Runtime presets are deprecated.');
+    output.warn('Use `bellwether init --preset <name>` to create a config file instead.\n');
+    if (loadPreset(name)) {
+        output.error(`Preset '${name}' already exists. Use 'preset update' to modify it.`);
+        process.exit(1);
+    }
+    const preset = {
+        name,
+        llm: {
+            provider: options.provider,
+            model: options.model,
+        },
+        interview: {
+            maxQuestionsPerTool: options.maxQuestions ? parseInt(options.maxQuestions, 10) : undefined,
+            personas: options.personas ? options.personas.split(',').map((s) => s.trim()) : undefined,
+        },
+        output: {
+            format: options.format,
+        },
+    };
+    savePreset(preset);
+    output.info(`Preset '${name}' created.`);
+    if (options.use) {
+        setCurrentPreset(name);
+        output.info(`Set '${name}' as current preset.`);
+    }
+}))
+    .addCommand(new Command('list')
+    .description('List all presets')
+    .action(() => {
+    const presets = listPresets();
+    const current = getCurrentPreset();
+    if (presets.length === 0) {
+        output.info('No presets found.');
+        output.info('Create one with: bellwether preset create <name>');
+        return;
+    }
+    output.info('Available presets:\n');
+    for (const name of presets) {
+        const marker = name === current ? ' (current)' : '';
+        const preset = loadPreset(name);
+        if (preset) {
+            output.info(`  ${name}${marker}`);
+            output.info(`    Provider: ${preset.llm.provider}`);
+            if (preset.llm.model) {
+                output.info(`    Model: ${preset.llm.model}`);
+            }
+        }
+    }
+}))
+    .addCommand(new Command('use')
+    .description('Set the current preset (deprecated)')
+    .argument('<name>', 'Preset name')
+    .action((name) => {
+    output.warn('Warning: Runtime presets are deprecated.');
+    output.warn('The test command now reads settings from bellwether.yaml.');
+    output.warn('Use `bellwether init --preset <name>` to create a config file.\n');
+    if (!loadPreset(name)) {
+        output.error(`Preset '${name}' not found.`);
+        output.info('Available presets: ' + (listPresets().join(', ') || 'none'));
+        process.exit(1);
+    }
+    setCurrentPreset(name);
+    output.info(`Set '${name}' as current preset.`);
+}))
+    .addCommand(new Command('show')
+    .description('Show preset details')
+    .argument('[name]', 'Preset name (defaults to current)')
+    .action((name) => {
+    const presetName = name ?? getCurrentPreset();
+    if (!presetName) {
+        output.error('No preset specified and no current preset set.');
+        process.exit(1);
+    }
+    const preset = loadPreset(presetName);
+    if (!preset) {
+        output.error(`Preset '${presetName}' not found.`);
+        process.exit(1);
+    }
+    output.info(`Preset: ${presetName}`);
+    output.info(stringify(preset));
+}))
+    .addCommand(new Command('delete')
+    .description('Delete a preset')
+    .argument('<name>', 'Preset name')
+    .action((name) => {
+    if (!deletePreset(name)) {
+        output.error(`Preset '${name}' not found.`);
+        process.exit(1);
+    }
+    // Clear current if deleted
+    if (getCurrentPreset() === name) {
+        setCurrentPreset(null);
+    }
+    output.info(`Preset '${name}' deleted.`);
+}))
+    .addCommand(new Command('update')
+    .description('Update an existing preset')
+    .argument('<name>', 'Preset name')
+    .option('-p, --provider <provider>', 'LLM provider')
+    .option('-m, --model <model>', 'LLM model')
+    .option('-q, --max-questions <n>', 'Max questions per tool')
+    .option('--personas <list>', 'Comma-separated list of personas')
+    .option('-f, --format <format>', 'Output format')
+    .action((name, options) => {
+    const preset = loadPreset(name);
+    if (!preset) {
+        output.error(`Preset '${name}' not found.`);
+        process.exit(1);
+    }
+    if (options.provider) {
+        preset.llm.provider = options.provider;
+    }
+    if (options.model) {
+        preset.llm.model = options.model;
+    }
+    if (options.maxQuestions) {
+        preset.interview.maxQuestionsPerTool = parseInt(options.maxQuestions, 10);
+    }
+    if (options.personas) {
+        preset.interview.personas = options.personas.split(',').map((s) => s.trim());
+    }
+    if (options.format) {
+        preset.output.format = options.format;
+    }
+    savePreset(preset);
+    output.info(`Preset '${name}' updated.`);
+}))
+    .addCommand(new Command('export')
+    .description('Export preset as YAML')
+    .argument('[name]', 'Preset name (defaults to current)')
+    .action((name) => {
+    const presetName = name ?? getCurrentPreset();
+    if (!presetName) {
+        output.error('No preset specified and no current preset set.');
+        process.exit(1);
+    }
+    const preset = loadPreset(presetName);
+    if (!preset) {
+        output.error(`Preset '${presetName}' not found.`);
+        process.exit(1);
+    }
+    output.info(stringify(preset));
+}))
+    .addCommand(new Command('import')
+    .description('Import preset from YAML file')
+    .argument('<file>', 'YAML file path')
+    .option('-n, --name <name>', 'Override preset name')
+    .action((file, options) => {
+    if (!existsSync(file)) {
+        output.error(`File not found: ${file}`);
+        process.exit(1);
+    }
+    const content = readFileSync(file, 'utf-8');
+    const rawPreset = parseYamlSecure(content);
+    // Validate preset against schema to prevent malicious YAML
+    const parseResult = presetSchema.safeParse({
+        ...rawPreset,
+        name: options.name || rawPreset?.name,
+    });
+    if (!parseResult.success) {
+        output.error('Invalid preset format:');
+        for (const issue of parseResult.error.issues) {
+            output.error(`  - ${issue.path.join('.')}: ${issue.message}`);
+        }
+        process.exit(1);
+    }
+    const preset = parseResult.data;
+    savePreset(preset);
+    output.info(`Preset '${preset.name}' imported.`);
+}));
+/**
+ * Get the preset config to merge with main config.
+ * Returns null if no current preset is set.
+ */
+export function getActivePresetConfig() {
+    const current = getCurrentPreset();
+    if (!current)
+        return null;
+    const preset = loadPreset(current);
+    if (!preset)
+        return null;
+    return {
+        llm: preset.llm,
+        interview: preset.interview,
+        output: preset.output,
+    };
+}
+//# sourceMappingURL=preset.js.map

package/dist/cli/commands/profile.d.ts

@@ -0,0 +1,33 @@
+import { Command } from 'commander';
+/**
+ * Profile configuration structure.
+ */
+interface Profile {
+    name: string;
+    llm: {
+        provider: 'openai' | 'anthropic' | 'ollama';
+        model?: string;
+    };
+    interview: {
+        maxQuestionsPerTool?: number;
+        personas?: string[];
+        timeout?: number;
+        skipErrorTests?: boolean;
+    };
+    output: {
+        format?: 'markdown' | 'json' | 'both';
+        outputDir?: string;
+    };
+}
+export declare const profileCommand: Command;
+/**
+ * Get the profile config to merge with main config.
+ * Returns null if no current profile is set.
+ */
+export declare function getActiveProfileConfig(): Partial<{
+    llm: Profile['llm'];
+    interview: Profile['interview'];
+    output: Profile['output'];
+}> | null;
+export {};
+//# sourceMappingURL=profile.d.ts.map

package/dist/cli/commands/profile.js

@@ -0,0 +1,286 @@
+import { Command } from 'commander';
+import { existsSync, mkdirSync, writeFileSync, readFileSync, readdirSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+import { stringify } from 'yaml';
+import { z } from 'zod';
+import { parseYamlSecure } from '../../utils/yaml-parser.js';
+import * as output from '../output.js';
+/**
+ * Zod schema for profile validation to prevent untrusted YAML injection.
+ */
+const profileSchema = z.object({
+    name: z.string().min(1).max(64),
+    llm: z.object({
+        provider: z.enum(['openai', 'anthropic', 'ollama']),
+        model: z.string().optional(),
+    }),
+    interview: z.object({
+        maxQuestionsPerTool: z.number().positive().optional(),
+        personas: z.array(z.string()).optional(),
+        timeout: z.number().positive().optional(),
+        skipErrorTests: z.boolean().optional(),
+    }).optional().default({}),
+    output: z.object({
+        format: z.enum(['markdown', 'json', 'both']).optional(),
+        outputDir: z.string().optional(),
+    }).optional().default({}),
+});
+const PROFILES_DIR = join(homedir(), '.bellwether', 'profiles');
+const CURRENT_PROFILE_FILE = join(homedir(), '.bellwether', 'current-profile');
+/**
+ * Validate profile name to prevent path traversal attacks.
+ * Only alphanumeric characters, hyphens, and underscores are allowed.
+ */
+function isValidProfileName(name) {
+    return /^[a-zA-Z0-9_-]+$/.test(name) && name.length > 0 && name.length <= 64;
+}
+function ensureProfilesDir() {
+    if (!existsSync(PROFILES_DIR)) {
+        mkdirSync(PROFILES_DIR, { recursive: true });
+    }
+}
+function getProfilePath(name) {
+    if (!isValidProfileName(name)) {
+        throw new Error(`Invalid profile name: "${name}". Only alphanumeric characters, hyphens, and underscores are allowed.`);
+    }
+    return join(PROFILES_DIR, `${name}.yaml`);
+}
+function loadProfile(name) {
+    const path = getProfilePath(name);
+    if (!existsSync(path)) {
+        return null;
+    }
+    const content = readFileSync(path, 'utf-8');
+    return parseYamlSecure(content);
+}
+function saveProfile(profile) {
+    ensureProfilesDir();
+    const path = getProfilePath(profile.name);
+    writeFileSync(path, stringify(profile));
+}
+function deleteProfile(name) {
+    const path = getProfilePath(name);
+    if (!existsSync(path)) {
+        return false;
+    }
+    unlinkSync(path);
+    return true;
+}
+function listProfiles() {
+    ensureProfilesDir();
+    return readdirSync(PROFILES_DIR)
+        .filter(f => f.endsWith('.yaml'))
+        .map(f => f.replace('.yaml', ''));
+}
+function getCurrentProfile() {
+    if (!existsSync(CURRENT_PROFILE_FILE)) {
+        return null;
+    }
+    return readFileSync(CURRENT_PROFILE_FILE, 'utf-8').trim();
+}
+function setCurrentProfile(name) {
+    ensureProfilesDir();
+    if (name) {
+        writeFileSync(CURRENT_PROFILE_FILE, name);
+    }
+    else if (existsSync(CURRENT_PROFILE_FILE)) {
+        unlinkSync(CURRENT_PROFILE_FILE);
+    }
+}
+export const profileCommand = new Command('profile')
+    .description('Manage interview profiles')
+    .addCommand(new Command('create')
+    .description('Create a new profile')
+    .argument('<name>', 'Profile name')
+    .option('-p, --provider <provider>', 'LLM provider (openai, anthropic, ollama)', 'openai')
+    .option('-m, --model <model>', 'LLM model')
+    .option('-q, --max-questions <n>', 'Max questions per tool')
+    .option('--personas <list>', 'Comma-separated list of personas')
+    .option('-f, --format <format>', 'Output format (markdown, json, both)')
+    .option('--use', 'Set as current profile after creation')
+    .action((name, options) => {
+    if (loadProfile(name)) {
+        output.error(`Profile '${name}' already exists. Use 'profile update' to modify it.`);
+        process.exit(1);
+    }
+    const profile = {
+        name,
+        llm: {
+            provider: options.provider,
+            model: options.model,
+        },
+        interview: {
+            maxQuestionsPerTool: options.maxQuestions ? parseInt(options.maxQuestions, 10) : undefined,
+            personas: options.personas ? options.personas.split(',').map((s) => s.trim()) : undefined,
+        },
+        output: {
+            format: options.format,
+        },
+    };
+    saveProfile(profile);
+    output.info(`Profile '${name}' created.`);
+    if (options.use) {
+        setCurrentProfile(name);
+        output.info(`Set '${name}' as current profile.`);
+    }
+}))
+    .addCommand(new Command('list')
+    .description('List all profiles')
+    .action(() => {
+    const profiles = listProfiles();
+    const current = getCurrentProfile();
+    if (profiles.length === 0) {
+        output.info('No profiles found.');
+        output.info('Create one with: bellwether profile create <name>');
+        return;
+    }
+    output.info('Available profiles:\n');
+    for (const name of profiles) {
+        const marker = name === current ? ' (current)' : '';
+        const profile = loadProfile(name);
+        if (profile) {
+            output.info(`  ${name}${marker}`);
+            output.info(`    Provider: ${profile.llm.provider}`);
+            if (profile.llm.model) {
+                output.info(`    Model: ${profile.llm.model}`);
+            }
+        }
+    }
+}))
+    .addCommand(new Command('use')
+    .description('Set the current profile')
+    .argument('<name>', 'Profile name')
+    .action((name) => {
+    if (!loadProfile(name)) {
+        output.error(`Profile '${name}' not found.`);
+        output.info('Available profiles: ' + (listProfiles().join(', ') || 'none'));
+        process.exit(1);
+    }
+    setCurrentProfile(name);
+    output.info(`Set '${name}' as current profile.`);
+}))
+    .addCommand(new Command('show')
+    .description('Show profile details')
+    .argument('[name]', 'Profile name (defaults to current)')
+    .action((name) => {
+    const profileName = name ?? getCurrentProfile();
+    if (!profileName) {
+        output.error('No profile specified and no current profile set.');
+        process.exit(1);
+    }
+    const profile = loadProfile(profileName);
+    if (!profile) {
+        output.error(`Profile '${profileName}' not found.`);
+        process.exit(1);
+    }
+    output.info(`Profile: ${profileName}`);
+    output.info(stringify(profile));
+}))
+    .addCommand(new Command('delete')
+    .description('Delete a profile')
+    .argument('<name>', 'Profile name')
+    .action((name) => {
+    if (!deleteProfile(name)) {
+        output.error(`Profile '${name}' not found.`);
+        process.exit(1);
+    }
+    // Clear current if deleted
+    if (getCurrentProfile() === name) {
+        setCurrentProfile(null);
+    }
+    output.info(`Profile '${name}' deleted.`);
+}))
+    .addCommand(new Command('update')
+    .description('Update an existing profile')
+    .argument('<name>', 'Profile name')
+    .option('-p, --provider <provider>', 'LLM provider')
+    .option('-m, --model <model>', 'LLM model')
+    .option('-q, --max-questions <n>', 'Max questions per tool')
+    .option('--personas <list>', 'Comma-separated list of personas')
+    .option('-f, --format <format>', 'Output format')
+    .action((name, options) => {
+    const profile = loadProfile(name);
+    if (!profile) {
+        output.error(`Profile '${name}' not found.`);
+        process.exit(1);
+    }
+    if (options.provider) {
+        profile.llm.provider = options.provider;
+    }
+    if (options.model) {
+        profile.llm.model = options.model;
+    }
+    if (options.maxQuestions) {
+        profile.interview.maxQuestionsPerTool = parseInt(options.maxQuestions, 10);
+    }
+    if (options.personas) {
+        profile.interview.personas = options.personas.split(',').map((s) => s.trim());
+    }
+    if (options.format) {
+        profile.output.format = options.format;
+    }
+    saveProfile(profile);
+    output.info(`Profile '${name}' updated.`);
+}))
+    .addCommand(new Command('export')
+    .description('Export profile as YAML')
+    .argument('[name]', 'Profile name (defaults to current)')
+    .action((name) => {
+    const profileName = name ?? getCurrentProfile();
+    if (!profileName) {
+        output.error('No profile specified and no current profile set.');
+        process.exit(1);
+    }
+    const profile = loadProfile(profileName);
+    if (!profile) {
+        output.error(`Profile '${profileName}' not found.`);
+        process.exit(1);
+    }
+    output.info(stringify(profile));
+}))
+    .addCommand(new Command('import')
+    .description('Import profile from YAML file')
+    .argument('<file>', 'YAML file path')
+    .option('-n, --name <name>', 'Override profile name')
+    .action((file, options) => {
+    if (!existsSync(file)) {
+        output.error(`File not found: ${file}`);
+        process.exit(1);
+    }
+    const content = readFileSync(file, 'utf-8');
+    const rawProfile = parseYamlSecure(content);
+    // Validate profile against schema to prevent malicious YAML
+    const parseResult = profileSchema.safeParse({
+        ...rawProfile,
+        name: options.name || rawProfile?.name,
+    });
+    if (!parseResult.success) {
+        output.error('Invalid profile format:');
+        for (const issue of parseResult.error.issues) {
+            output.error(`  - ${issue.path.join('.')}: ${issue.message}`);
+        }
+        process.exit(1);
+    }
+    const profile = parseResult.data;
+    saveProfile(profile);
+    output.info(`Profile '${profile.name}' imported.`);
+}));
+/**
+ * Get the profile config to merge with main config.
+ * Returns null if no current profile is set.
+ */
+export function getActiveProfileConfig() {
+    const current = getCurrentProfile();
+    if (!current)
+        return null;
+    const profile = loadProfile(current);
+    if (!profile)
+        return null;
+    return {
+        llm: profile.llm,
+        interview: profile.interview,
+        output: profile.output,
+    };
+}
+//# sourceMappingURL=profile.js.map

package/dist/cli/commands/registry.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Registry command - search and lookup MCP servers from the registry.
+ */
+import { Command } from 'commander';
+/**
+ * Create a new registry command instance.
+ * Useful for testing where fresh command instances are needed.
+ */
+export declare function createRegistryCommand(): Command;
+export declare const registryCommand: Command;
+//# sourceMappingURL=registry.d.ts.map