@dotsetlabs/bellwether 0.10.0

package/dist/config/template.js

@@ -0,0 +1,647 @@
+/**
+ * Configuration templates for bellwether init.
+ *
+ * These templates include ALL possible options with comments,
+ * making the config file self-documenting.
+ */
+import { CONFIG_DEFAULTS } from './defaults.js';
+function formatYamlList(values, indent = 4) {
+    const pad = ' '.repeat(indent);
+    return values.map((value) => `${pad}- ${value}`).join('\n');
+}
+/**
+ * Generate a comprehensive bellwether.yaml configuration file.
+ *
+ * The generated file includes ALL possible options with explanatory comments,
+ * so users can customize without consulting documentation.
+ *
+ * This config is used by both 'bellwether check' and 'bellwether explore' commands.
+ */
+export function generateConfigTemplate(options = {}) {
+    const { serverCommand = '', serverArgs = [], provider = 'ollama', preset, envVars = [], ciOptimized = false, } = options;
+    const defaults = CONFIG_DEFAULTS;
+    const serverArgsYaml = serverArgs.length > 0
+        ? `\n  args:\n${serverArgs.map(arg => `    - "${arg}"`).join('\n')}`
+        : '\n  args: []';
+    const presetComment = preset ? `# Generated with: bellwether init --preset ${preset}\n` : '';
+    // Generate env section if env vars were detected
+    const envVarsYaml = envVars.length > 0
+        ? `\n  env:\n${envVars.map(v => `    ${v}: "\${${v}}"`).join('\n')}`
+        : '';
+    return `# Bellwether Configuration
+# Generated by: bellwether init
+# Docs: https://docs.bellwether.sh/guides/configuration
+${presetComment}
+# This config is used by both commands:
+#   bellwether check   - Schema validation and drift detection (free, fast)
+#   bellwether explore - LLM-powered behavioral exploration
+
+# =============================================================================
+# SERVER (used by both commands)
+# =============================================================================
+# The MCP server to validate/explore.
+
+server:
+  # Command to start the MCP server
+  command: "${serverCommand}"
+  # Arguments to pass to the server command${serverArgsYaml}
+
+  # Timeout for server startup and tool calls (milliseconds, default: ${defaults.server.timeout})
+  timeout: ${defaults.server.timeout}
+
+  # Additional environment variables for the server process
+  # Use \${VAR} syntax to reference environment variables${envVarsYaml}${envVars.length === 0 ? `
+  # env:
+  #   NODE_ENV: production
+  #   API_KEY: "\${API_KEY}"` : ''}
+
+# =============================================================================
+# SCENARIOS (used by both commands)
+# =============================================================================
+# Custom deterministic test scenarios for reproducible testing.
+
+scenarios:
+  # Path to scenarios YAML file
+  # path: "./bellwether-tests.yaml"
+
+  # Run ONLY scenarios (skip generated tests)
+  only: ${defaults.scenarios.only}
+
+# =============================================================================
+# OUTPUT (used by both commands)
+# =============================================================================
+# bellwether outputs: documentation + JSON reports (configurable via output.files)
+
+output:
+  # Directory for JSON output files (see output.files)
+  dir: "${defaults.output.dir}"
+
+  # Directory for documentation files (CONTRACT.md, AGENTS.md)
+  # These are kept in root by default for visibility
+  docsDir: "${defaults.output.docsDir}"
+
+  # Output format: agents.md (markdown only), json (JSON only), or both
+  format: ${defaults.output.format}
+
+  # Example output settings (for CONTRACT.md and AGENTS.md)
+  examples:
+    # Include full (non-truncated) examples in documentation
+    full: ${defaults.output.examples.full}
+
+    # Maximum example length in characters (100-50000)
+    maxLength: ${defaults.output.examples.maxLength}
+
+    # Maximum examples per tool (1-20)
+    maxPerTool: ${defaults.output.examples.maxPerTool}
+
+  # Output file names (relative to output dirs above)
+  files:
+    checkReport: ${defaults.output.files.checkReport}
+    exploreReport: ${defaults.output.files.exploreReport}
+    contractDoc: ${defaults.output.files.contractDoc}
+    agentsDoc: ${defaults.output.files.agentsDoc}
+    verificationReport: ${defaults.output.files.verificationReport}
+
+# =============================================================================
+# CHECK COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether check' - schema validation and drift detection.
+# Free, fast, deterministic. No LLM required.
+
+baseline:
+  # Default baseline file path (relative to output.dir or absolute)
+  path: ${defaults.baseline.path}
+
+  # Auto-save baseline after each check (set path to enable)
+  # savePath: "./bellwether-baseline.json"
+
+  # Path to baseline for drift comparison (enables drift detection)
+  # comparePath: "./bellwether-baseline.json"
+
+  # Fail if drift is detected (useful for CI)
+  failOnDrift: ${ciOptimized ? 'true' : 'false'}
+
+  # Default output format for baseline compare/diff: text, json, markdown, compact
+  outputFormat: ${defaults.baseline.outputFormat}
+
+  # Severity thresholds for filtering and CI failure
+  severity:
+    # Minimum severity to include in reports: none, info, warning, breaking
+    minimumSeverity: ${defaults.baseline.severity.minimumSeverity}
+
+    # Severity at which to fail CI checks: none, info, warning, breaking
+    failOnSeverity: ${defaults.baseline.severity.failOnSeverity}
+
+    # Suppress warning-level changes from output
+    suppressWarnings: ${defaults.baseline.severity.suppressWarnings}
+
+    # Custom severity overrides per aspect (uncomment to use)
+    # aspectOverrides:
+    #   description: none        # Ignore description-only changes
+    #   response_structure: info # Downgrade response structure to info
+
+check:
+  # Enable incremental checking (only test tools with changed schemas)
+  # Requires an existing baseline for comparison
+  incremental: ${defaults.check.incremental}
+
+  # Maximum age of cached results in hours (for incremental checking)
+  # Default: 168 (1 week), Range: 1-720 (30 days)
+  incrementalCacheHours: ${defaults.check.incrementalCacheHours}
+
+  # Enable parallel tool testing for faster checks
+  # Recommended for servers with multiple tools
+  parallel: ${defaults.check.parallel}
+
+  # Number of concurrent tool workers (1-10)
+  # Higher values = faster but more server load
+  parallelWorkers: ${defaults.check.parallelWorkers}
+
+  # Performance regression threshold percentage (0-100)
+  # Tool is flagged if p50 latency increases by more than this percentage
+  performanceThreshold: ${defaults.check.performanceThreshold}
+
+  # Diff output format: text, json, compact, github, markdown, junit, sarif
+  diffFormat: ${defaults.check.diffFormat}
+
+  # Number of warmup runs before timing samples (0-5)
+  # First sample(s) are excluded from variance calculation to reduce cold-start noise
+  warmupRuns: ${defaults.check.warmupRuns}
+
+  # Enable smart test value generation from schema descriptions
+  # Parses descriptions for format hints (e.g., "YYYY-MM-DD" generates valid dates)
+  smartTestValues: ${defaults.check.smartTestValues}
+
+  # Stateful testing (create -> use -> delete) for tool dependencies
+  statefulTesting:
+    enabled: ${defaults.check.statefulTesting.enabled}
+    maxChainLength: ${defaults.check.statefulTesting.maxChainLength}
+    shareOutputsBetweenTools: ${defaults.check.statefulTesting.shareOutputsBetweenTools}
+
+  # External service handling for tools that need credentials
+  externalServices:
+    # Mode for unconfigured services: skip | mock | fail
+    mode: ${defaults.check.externalServices.mode}
+    # Per-service overrides (optional)
+    # services:
+    #   plaid:
+    #     enabled: false
+    #     sandboxCredentials:
+    #       clientId: "\${PLAID_CLIENT_ID}"
+    #       secret: "\${PLAID_SECRET}"
+
+  # Response assertions (semantic validation of responses)
+  assertions:
+    enabled: ${defaults.check.assertions.enabled}
+    strict: ${defaults.check.assertions.strict}
+    infer: ${defaults.check.assertions.infer}
+
+  # Rate limiting controls
+  rateLimit:
+    enabled: ${defaults.check.rateLimit.enabled}
+    requestsPerSecond: ${defaults.check.rateLimit.requestsPerSecond}
+    burstLimit: ${defaults.check.rateLimit.burstLimit}
+    backoffStrategy: ${defaults.check.rateLimit.backoffStrategy}
+    maxRetries: ${defaults.check.rateLimit.maxRetries}
+
+  # Security testing settings
+  security:
+    # Enable security vulnerability testing
+    # Tests for SQL injection, XSS, path traversal, command injection, SSRF
+    enabled: ${defaults.check.security.enabled}
+
+    # Security categories to test (when enabled)
+    categories:
+${formatYamlList(defaults.check.security.categories, 6)}
+
+  # Statistical sampling settings for performance baselines
+  sampling:
+    # Minimum samples per tool (1-50, default: 10)
+    # Higher values = more reliable baselines but longer test runs
+    minSamples: ${defaults.check.sampling.minSamples}
+
+    # Target confidence level: low (1+ samples), medium (5+), high (10+)
+    targetConfidence: ${defaults.check.sampling.targetConfidence}
+
+    # Fail if confidence is below target (useful for CI)
+    failOnLowConfidence: ${defaults.check.sampling.failOnLowConfidence}
+
+  # Metrics configuration
+  metrics:
+    # Count validation rejections as successes in reliability metrics
+    countValidationAsSuccess: ${defaults.check.metrics.countValidationAsSuccess}
+    # Separate validation metrics from happy-path reliability
+    separateValidationMetrics: ${defaults.check.metrics.separateValidationMetrics}
+
+# =============================================================================
+# EXPLORE COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether explore' - LLM-powered behavioral exploration.
+# Requires LLM provider (Ollama is free and local).
+
+llm:
+  # Provider: ollama (local, free), openai, or anthropic
+  provider: ${provider}
+
+  # Model to use. Leave empty for provider default.
+  # Defaults: ollama=qwen3:8b, openai=gpt-4.1-nano, anthropic=claude-haiku-4-5
+  model: "${defaults.llm.model}"
+
+  # Ollama settings (for local LLM)
+  ollama:
+    baseUrl: "${defaults.llm.ollama.baseUrl}"
+
+  # API keys are loaded from environment variables or 'bellwether auth'.
+  # SECURITY: Never put API keys directly in this file!
+
+explore:
+  # Personas provide different testing perspectives:
+  # - technical_writer: Documentation quality, clarity, completeness
+  # - security_tester: Security vulnerabilities, input validation
+  # - qa_engineer: Edge cases, error handling, reliability
+  # - novice_user: Usability, confusing behavior, missing guidance
+  personas:
+${formatYamlList(defaults.explore.personas, 4)}
+
+  # Maximum questions to ask per tool (1-10)
+  maxQuestionsPerTool: ${defaults.explore.maxQuestionsPerTool}
+
+  # Run personas in parallel for faster execution
+  parallelPersonas: ${defaults.explore.parallelPersonas}
+
+  # Maximum concurrent persona interviews (1-10)
+  personaConcurrency: ${defaults.explore.personaConcurrency}
+
+  # Skip error/edge case testing
+  skipErrorTests: ${defaults.explore.skipErrorTests}
+
+workflows:
+  # Path to workflow definitions YAML file
+  # path: "./bellwether-workflows.yaml"
+
+  # Enable LLM-based workflow discovery
+  discover: ${defaults.workflows.discover}
+
+  # Track state changes between workflow steps
+  trackState: ${defaults.workflows.trackState}
+
+  # Auto-generate workflows from discovered tools
+  autoGenerate: ${defaults.workflows.autoGenerate}
+
+  # Skip steps whose dependencies (previous steps providing data) have failed
+  # When true, a step that depends on data from a failed step will be skipped
+  requireSuccessfulDependencies: ${defaults.workflows.requireSuccessfulDependencies}
+
+  # Timeout per workflow step in milliseconds (1000-300000)
+  stepTimeout: ${defaults.workflows.stepTimeout}
+
+  # Timeout configuration for workflow operations (ms)
+  timeouts:
+    toolCall: ${defaults.workflows.timeouts.toolCall}
+    stateSnapshot: ${defaults.workflows.timeouts.stateSnapshot}
+    probeTool: ${defaults.workflows.timeouts.probeTool}
+    llmAnalysis: ${defaults.workflows.timeouts.llmAnalysis}
+    llmSummary: ${defaults.workflows.timeouts.llmSummary}
+
+# =============================================================================
+# WATCH MODE SETTINGS
+# =============================================================================
+# Settings for 'bellwether watch' - continuous drift monitoring.
+
+watch:
+  # Path to watch for file changes
+  path: "${defaults.watch.path}"
+
+  # Polling interval in milliseconds (1000-60000)
+  interval: ${defaults.watch.interval}
+
+  # File extensions to watch for changes
+  extensions:
+${formatYamlList(defaults.watch.extensions, 4)}
+
+  # Command to run when drift is detected (optional)
+  # onDrift: "npm test"
+
+# =============================================================================
+# COMMON SETTINGS
+# =============================================================================
+
+cache:
+  # Enable response caching (recommended)
+  enabled: ${defaults.cache.enabled}
+
+  # Cache directory
+  dir: "${defaults.cache.dir}"
+
+logging:
+  # Log level: debug, info, warn, error, silent
+  level: ${defaults.logging.level}
+
+  # Show verbose output
+  verbose: ${defaults.logging.verbose}
+
+# =============================================================================
+# DISCOVER COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether discover' - capability discovery.
+
+discovery:
+  # Output as JSON
+  json: ${defaults.discovery.json}
+
+  # Connection timeout in milliseconds
+  timeout: ${defaults.discovery.timeout}
+
+  # Transport type: stdio, sse, streamable-http
+  transport: ${defaults.discovery.transport}
+
+  # Remote MCP server URL (required for sse/streamable-http)
+  # url: "https://example.com/mcp"
+
+  # Session ID for remote server authentication
+  # sessionId: "session-id"
+
+# =============================================================================
+# REGISTRY COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether registry' - server lookup.
+
+registry:
+  # Maximum results to show
+  limit: ${defaults.registry.limit}
+
+  # Output as JSON
+  json: ${defaults.registry.json}
+
+# =============================================================================
+# HISTORY COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether history' and 'bellwether diff' (cloud history).
+
+history:
+  # Number of versions to show
+  limit: ${defaults.history.limit}
+
+  # Output as JSON
+  json: ${defaults.history.json}
+
+# =============================================================================
+# LINK COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether link' (cloud project linking).
+
+link:
+  # Default server command when creating new projects
+  defaultServerCommand: "${defaults.link.defaultServerCommand}"
+
+# =============================================================================
+# GOLDEN COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether golden' output validation.
+
+golden:
+  # Default JSON args for golden save
+  defaultArgs: '${defaults.golden.defaultArgs}'
+
+  # Default comparison mode: exact, structural, semantic
+  mode: ${defaults.golden.mode}
+
+  # Output format for compare: text, json, markdown
+  compareFormat: ${defaults.golden.compareFormat}
+
+  # Output format for list: text, json
+  listFormat: ${defaults.golden.listFormat}
+
+  # Normalize timestamps by default
+  normalizeTimestamps: ${defaults.golden.normalizeTimestamps}
+
+  # Normalize UUIDs by default
+  normalizeUuids: ${defaults.golden.normalizeUuids}
+
+# =============================================================================
+# VERIFY COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether verify' verification reports.
+
+verify:
+  # Default verification tier: bronze, silver, gold, platinum
+  tier: ${defaults.verify.tier}
+
+  # Include security testing by default
+  security: ${defaults.verify.security}
+
+  # Output as JSON by default
+  json: ${defaults.verify.json}
+
+  # Output badge URL only by default
+  badgeOnly: ${defaults.verify.badgeOnly}
+
+# =============================================================================
+# CONTRACT COMMAND SETTINGS
+# =============================================================================
+# Settings for 'bellwether contract' - contract validation/generation.
+
+contract:
+  # Default contract file path
+  # path: "./contract.bellwether.yaml"
+
+  # Validation mode: strict, lenient, report
+  mode: ${defaults.contract.mode}
+
+  # Output format: text, json, markdown
+  format: ${defaults.contract.format}
+
+  # Server startup timeout in milliseconds
+  timeout: ${defaults.contract.timeout}
+
+  # Exit with error when violations are found
+  failOnViolation: ${defaults.contract.failOnViolation}
+`;
+}
+/**
+ * Preset configurations for common use cases.
+ */
+export const PRESETS = {
+    /**
+     * CI preset: Optimized for 'bellwether check' in CI/CD pipelines.
+     * Fast, free, deterministic with failOnDrift enabled.
+     */
+    ci: {
+        ciOptimized: true,
+        provider: 'ollama', // Not used by check, but sensible default
+    },
+    /**
+     * Security preset: Optimized for 'bellwether explore' with security focus.
+     */
+    security: {
+        provider: 'anthropic',
+    },
+    /**
+     * Thorough preset: Optimized for 'bellwether explore' with all personas.
+     */
+    thorough: {
+        provider: 'anthropic',
+    },
+    /**
+     * Local preset: Optimized for 'bellwether explore' with local Ollama.
+     * Free, private, no API keys needed.
+     */
+    local: {
+        provider: 'ollama',
+    },
+};
+/**
+ * Get preset-specific YAML overrides for documentation purposes.
+ *
+ * @param presetName - Name of the preset (ci, security, thorough, local)
+ * @returns YAML string with preset-specific settings or null if preset not found
+ */
+export function getPresetOverrides(presetName) {
+    const preset = PRESETS[presetName];
+    if (!preset)
+        return null;
+    // Return additional YAML to append/override based on preset
+    switch (presetName) {
+        case 'ci':
+            return `
+# CI Preset Overrides
+# Optimized for 'bellwether check' in CI/CD pipelines
+
+baseline:
+  failOnDrift: true
+  severity:
+    failOnSeverity: breaking
+
+check:
+  parallel: true
+  parallelWorkers: 4
+  performanceThreshold: 10
+  sampling:
+    minSamples: 5
+    targetConfidence: medium
+    failOnLowConfidence: true
+
+output:
+  examples:
+    full: false
+    maxLength: 1000
+
+logging:
+  level: warn
+`;
+        case 'security':
+            return `
+# Security Preset Overrides
+# Optimized for security-focused testing
+
+check:
+  security:
+    enabled: true
+    categories:
+      - sql_injection
+      - xss
+      - path_traversal
+      - command_injection
+      - ssrf
+  sampling:
+    minSamples: 5
+    targetConfidence: high
+
+llm:
+  provider: anthropic
+  model: claude-sonnet-4-5  # Better reasoning for security analysis
+
+explore:
+  personas:
+    - technical_writer
+    - security_tester
+    - qa_engineer
+  maxQuestionsPerTool: 5
+  skipErrorTests: false
+
+output:
+  examples:
+    full: true
+    maxLength: 5000
+`;
+        case 'thorough':
+            return `
+# Thorough Preset Overrides
+# Comprehensive testing with all features enabled
+
+check:
+  parallel: true
+  parallelWorkers: 6
+  security:
+    enabled: true
+    categories:
+      - sql_injection
+      - xss
+      - path_traversal
+      - command_injection
+      - ssrf
+  sampling:
+    minSamples: 10
+    targetConfidence: high
+    failOnLowConfidence: true
+
+llm:
+  provider: anthropic
+  model: claude-sonnet-4-5
+
+explore:
+  personas:
+    - technical_writer
+    - security_tester
+    - qa_engineer
+    - novice_user
+  maxQuestionsPerTool: 5
+  parallelPersonas: true
+  skipErrorTests: false
+
+workflows:
+  discover: true
+  trackState: true
+  autoGenerate: true
+
+output:
+  examples:
+    full: true
+    maxLength: 10000
+    maxPerTool: 10
+`;
+        case 'local':
+            return `
+# Local Preset Overrides
+# Exploration using local Ollama (free, private)
+
+llm:
+  provider: ollama
+  model: qwen3:8b
+  ollama:
+    baseUrl: "http://localhost:11434"
+
+explore:
+  personas:
+    - technical_writer
+  maxQuestionsPerTool: 3
+`;
+        default:
+            return null;
+    }
+}
+/**
+ * Generate a preset-specific configuration.
+ */
+export function generatePresetConfig(presetName, options = {}) {
+    const preset = PRESETS[presetName];
+    if (!preset) {
+        throw new Error(`Unknown preset: ${presetName}. Available: ${Object.keys(PRESETS).join(', ')}`);
+    }
+    return generateConfigTemplate({
+        ...preset,
+        ...options,
+        preset: presetName,
+    });
+}
+//# sourceMappingURL=template.js.map