@dotsetlabs/bellwether 0.10.0

package/dist/errors/types.js

@@ -0,0 +1,584 @@
+/**
+ * Comprehensive error types for Bellwether.
+ *
+ * Error hierarchy:
+ * - BellwetherError (base)
+ *   - TransportError (MCP communication)
+ *   - LLMError (LLM provider issues)
+ *   - InterviewError (interview execution)
+ *   - WorkflowError (workflow execution)
+ *   - ConfigError (configuration issues)
+ */
+/**
+ * Base error class for all Bellwether errors.
+ */
+export class BellwetherError extends Error {
+    /** Error code for programmatic handling */
+    code;
+    /** Error severity */
+    severity;
+    /** Whether this error is retryable */
+    retryable;
+    /** Error context for debugging */
+    context;
+    /** Original error if this wraps another */
+    cause;
+    constructor(message, options) {
+        super(message);
+        this.name = 'BellwetherError';
+        this.code = options.code;
+        this.severity = options.severity ?? 'medium';
+        this.retryable = options.retryable ?? 'unknown';
+        this.context = options.context ?? {};
+        this.cause = options.cause;
+        // Maintain proper stack trace
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, this.constructor);
+        }
+    }
+    /**
+     * Create a new error with additional context.
+     */
+    withContext(additionalContext) {
+        return new BellwetherError(this.message, {
+            code: this.code,
+            severity: this.severity,
+            retryable: this.retryable,
+            context: { ...this.context, ...additionalContext },
+            cause: this.cause,
+        });
+    }
+    /**
+     * Convert to JSON for logging.
+     */
+    toJSON() {
+        return {
+            name: this.name,
+            code: this.code,
+            message: this.message,
+            severity: this.severity,
+            retryable: this.retryable,
+            context: this.context,
+            cause: this.cause
+                ? {
+                    name: this.cause.name,
+                    message: this.cause.message,
+                }
+                : undefined,
+            stack: this.stack,
+        };
+    }
+}
+/**
+ * Base class for transport-related errors.
+ */
+export class TransportError extends BellwetherError {
+    constructor(message, options) {
+        super(message, options);
+        this.name = 'TransportError';
+    }
+}
+/**
+ * Connection failed or was lost.
+ */
+export class ConnectionError extends TransportError {
+    constructor(message, context, cause) {
+        super(message, {
+            code: 'TRANSPORT_CONNECTION_FAILED',
+            severity: 'high',
+            retryable: 'retryable',
+            context,
+            cause,
+        });
+        this.name = 'ConnectionError';
+    }
+}
+/**
+ * Request timed out.
+ */
+export class TimeoutError extends TransportError {
+    /** Timeout value in milliseconds */
+    timeoutMs;
+    constructor(message, timeoutMs, context, cause) {
+        super(message, {
+            code: 'TRANSPORT_TIMEOUT',
+            severity: 'medium',
+            retryable: 'retryable',
+            context: { ...context, metadata: { ...context?.metadata, timeoutMs } },
+            cause,
+        });
+        this.name = 'TimeoutError';
+        this.timeoutMs = timeoutMs;
+    }
+}
+/**
+ * Server process exited unexpectedly.
+ */
+export class ServerExitError extends TransportError {
+    /** Exit code if available */
+    exitCode;
+    /** Exit signal if available */
+    signal;
+    constructor(message, exitCode, signal, context) {
+        super(message, {
+            code: 'TRANSPORT_SERVER_EXIT',
+            severity: 'high',
+            retryable: 'terminal',
+            context: { ...context, metadata: { ...context?.metadata, exitCode, signal } },
+        });
+        this.name = 'ServerExitError';
+        this.exitCode = exitCode;
+        this.signal = signal;
+    }
+}
+/**
+ * Protocol error (invalid message format, etc).
+ */
+export class ProtocolError extends TransportError {
+    constructor(message, context, cause) {
+        super(message, {
+            code: 'TRANSPORT_PROTOCOL_ERROR',
+            severity: 'high',
+            retryable: 'terminal',
+            context,
+            cause,
+        });
+        this.name = 'ProtocolError';
+    }
+}
+/**
+ * Buffer overflow during message processing.
+ */
+export class BufferOverflowError extends TransportError {
+    /** Current buffer size */
+    bufferSize;
+    /** Maximum allowed size */
+    maxSize;
+    constructor(bufferSize, maxSize, context) {
+        super(`Buffer overflow: ${bufferSize} bytes exceeds maximum ${maxSize} bytes`, {
+            code: 'TRANSPORT_BUFFER_OVERFLOW',
+            severity: 'high',
+            retryable: 'terminal',
+            context: { ...context, metadata: { bufferSize, maxSize } },
+        });
+        this.name = 'BufferOverflowError';
+        this.bufferSize = bufferSize;
+        this.maxSize = maxSize;
+    }
+}
+/**
+ * Base class for LLM-related errors.
+ */
+export class LLMError extends BellwetherError {
+    /** LLM provider name */
+    provider;
+    /** Model name if available */
+    model;
+    constructor(message, provider, options) {
+        super(message, {
+            code: options.code,
+            severity: options.severity,
+            retryable: options.retryable,
+            context: {
+                ...options.context,
+                metadata: { ...options.context?.metadata, provider, model: options.model },
+            },
+            cause: options.cause,
+        });
+        this.name = 'LLMError';
+        this.provider = provider;
+        this.model = options.model;
+    }
+}
+/**
+ * Authentication/API key error.
+ */
+export class LLMAuthError extends LLMError {
+    constructor(provider, model, context, cause) {
+        super('LLM authentication failed - check API key', provider, {
+            code: 'LLM_AUTH_FAILED',
+            model,
+            severity: 'critical',
+            retryable: 'terminal',
+            context,
+            cause,
+        });
+        this.name = 'LLMAuthError';
+    }
+}
+/**
+ * Rate limit exceeded.
+ */
+export class LLMRateLimitError extends LLMError {
+    /** Retry after in milliseconds if known */
+    retryAfterMs;
+    constructor(provider, retryAfterMs, model, context, cause) {
+        super('LLM rate limit exceeded', provider, {
+            code: 'LLM_RATE_LIMITED',
+            model,
+            severity: 'medium',
+            retryable: 'retryable',
+            context: { ...context, metadata: { ...context?.metadata, retryAfterMs } },
+            cause,
+        });
+        this.name = 'LLMRateLimitError';
+        this.retryAfterMs = retryAfterMs;
+    }
+}
+/**
+ * Quota/credits exhausted.
+ */
+export class LLMQuotaError extends LLMError {
+    constructor(provider, model, context, cause) {
+        super('LLM quota or credits exhausted', provider, {
+            code: 'LLM_QUOTA_EXHAUSTED',
+            model,
+            severity: 'critical',
+            retryable: 'terminal',
+            context,
+            cause,
+        });
+        this.name = 'LLMQuotaError';
+    }
+}
+/**
+ * Model refused to complete request (content policy, etc).
+ */
+export class LLMRefusalError extends LLMError {
+    /** Refusal reason if available */
+    reason;
+    constructor(provider, reason, model, context, cause) {
+        super(reason ? `LLM refused request: ${reason}` : 'LLM refused request', provider, {
+            code: 'LLM_REFUSED',
+            model,
+            severity: 'medium',
+            retryable: 'terminal', // Different prompt might work, but same prompt won't
+            context: { ...context, metadata: { ...context?.metadata, reason } },
+            cause,
+        });
+        this.name = 'LLMRefusalError';
+        this.reason = reason;
+    }
+}
+/**
+ * LLM response parsing failed.
+ */
+export class LLMParseError extends LLMError {
+    /** Raw response that couldn't be parsed */
+    rawResponse;
+    constructor(provider, rawResponse, model, context, cause) {
+        super('Failed to parse LLM response', provider, {
+            code: 'LLM_PARSE_ERROR',
+            model,
+            severity: 'medium',
+            retryable: 'retryable', // Different response might parse
+            context: {
+                ...context,
+                metadata: {
+                    ...context?.metadata,
+                    rawResponsePreview: rawResponse?.slice(0, 200),
+                },
+            },
+            cause,
+        });
+        this.name = 'LLMParseError';
+        this.rawResponse = rawResponse;
+    }
+}
+/**
+ * LLM connection failed.
+ */
+export class LLMConnectionError extends LLMError {
+    constructor(provider, model, context, cause) {
+        super(`Failed to connect to LLM provider: ${provider}`, provider, {
+            code: 'LLM_CONNECTION_FAILED',
+            model,
+            severity: 'high',
+            retryable: 'retryable',
+            context,
+            cause,
+        });
+        this.name = 'LLMConnectionError';
+    }
+}
+/**
+ * Base class for interview-related errors.
+ */
+export class InterviewError extends BellwetherError {
+    constructor(message, options) {
+        super(message, options);
+        this.name = 'InterviewError';
+    }
+}
+/**
+ * Tool call failed.
+ */
+export class ToolCallError extends InterviewError {
+    /** Tool name */
+    toolName;
+    /** Arguments passed to tool */
+    args;
+    constructor(toolName, args, message, context, cause) {
+        super(`Tool call failed for '${toolName}': ${message}`, {
+            code: 'INTERVIEW_TOOL_CALL_FAILED',
+            severity: 'medium',
+            retryable: 'retryable',
+            context: { ...context, tool: toolName, metadata: { ...context?.metadata, args } },
+            cause,
+        });
+        this.name = 'ToolCallError';
+        this.toolName = toolName;
+        this.args = args;
+    }
+}
+/**
+ * Discovery failed.
+ */
+export class DiscoveryError extends InterviewError {
+    constructor(message, context, cause) {
+        super(`Server discovery failed: ${message}`, {
+            code: 'INTERVIEW_DISCOVERY_FAILED',
+            severity: 'high',
+            retryable: 'retryable',
+            context,
+            cause,
+        });
+        this.name = 'DiscoveryError';
+    }
+}
+/**
+ * Question generation failed.
+ */
+export class QuestionGenerationError extends InterviewError {
+    constructor(toolName, context, cause) {
+        super(`Failed to generate questions for tool '${toolName}'`, {
+            code: 'INTERVIEW_QUESTION_GEN_FAILED',
+            severity: 'low',
+            retryable: 'retryable',
+            context: { ...context, tool: toolName },
+            cause,
+        });
+        this.name = 'QuestionGenerationError';
+    }
+}
+/**
+ * Analysis/synthesis failed.
+ */
+export class AnalysisError extends InterviewError {
+    constructor(phase, context, cause) {
+        super(`Failed to analyze ${phase}`, {
+            code: `INTERVIEW_${phase.toUpperCase()}_ANALYSIS_FAILED`,
+            severity: 'low',
+            retryable: 'retryable',
+            context: { ...context, metadata: { ...context?.metadata, phase } },
+            cause,
+        });
+        this.name = 'AnalysisError';
+    }
+}
+/**
+ * Base class for workflow-related errors.
+ */
+export class WorkflowError extends BellwetherError {
+    /** Workflow ID */
+    workflowId;
+    /** Workflow name */
+    workflowName;
+    constructor(message, workflowId, options) {
+        super(message, {
+            code: options.code,
+            severity: options.severity,
+            retryable: options.retryable,
+            context: {
+                ...options.context,
+                workflow: workflowId,
+                metadata: { ...options.context?.metadata, workflowName: options.workflowName },
+            },
+            cause: options.cause,
+        });
+        this.name = 'WorkflowError';
+        this.workflowId = workflowId;
+        this.workflowName = options.workflowName;
+    }
+}
+/**
+ * Workflow step failed.
+ */
+export class WorkflowStepError extends WorkflowError {
+    /** Step index (0-based) */
+    stepIndex;
+    /** Tool name for this step */
+    toolName;
+    constructor(workflowId, stepIndex, toolName, message, workflowName, context, cause) {
+        super(`Workflow step ${stepIndex + 1} failed (${toolName}): ${message}`, workflowId, {
+            code: 'WORKFLOW_STEP_FAILED',
+            workflowName,
+            severity: 'medium',
+            retryable: 'retryable',
+            context: { ...context, stepIndex, tool: toolName },
+            cause,
+        });
+        this.name = 'WorkflowStepError';
+        this.stepIndex = stepIndex;
+        this.toolName = toolName;
+    }
+}
+/**
+ * Argument resolution failed (JSON path evaluation).
+ */
+export class ArgResolutionError extends WorkflowError {
+    /** The path expression that failed */
+    pathExpression;
+    /** Step index where resolution failed */
+    stepIndex;
+    constructor(workflowId, stepIndex, pathExpression, message, workflowName, context) {
+        super(`Argument resolution failed at step ${stepIndex + 1}: ${message}`, workflowId, {
+            code: 'WORKFLOW_ARG_RESOLUTION_FAILED',
+            workflowName,
+            severity: 'medium',
+            retryable: 'terminal', // Path expression won't change
+            context: {
+                ...context,
+                stepIndex,
+                metadata: { ...context?.metadata, pathExpression },
+            },
+        });
+        this.name = 'ArgResolutionError';
+        this.pathExpression = pathExpression;
+        this.stepIndex = stepIndex;
+    }
+}
+/**
+ * Assertion failed during workflow execution.
+ */
+export class AssertionError extends WorkflowError {
+    /** The assertion that failed */
+    assertion;
+    /** Actual value received */
+    actualValue;
+    constructor(workflowId, stepIndex, assertion, actualValue, workflowName, context) {
+        super(`Assertion failed at step ${stepIndex + 1}: ${assertion}`, workflowId, {
+            code: 'WORKFLOW_ASSERTION_FAILED',
+            workflowName,
+            severity: 'medium',
+            retryable: 'terminal', // Same input = same assertion failure
+            context: {
+                ...context,
+                stepIndex,
+                metadata: { ...context?.metadata, assertion, actualValue },
+            },
+        });
+        this.name = 'AssertionError';
+        this.assertion = assertion;
+        this.actualValue = actualValue;
+    }
+}
+/**
+ * Configuration-related error.
+ */
+export class ConfigError extends BellwetherError {
+    constructor(message, context, cause) {
+        super(message, {
+            code: 'CONFIG_ERROR',
+            severity: 'high',
+            retryable: 'terminal',
+            context,
+            cause,
+        });
+        this.name = 'ConfigError';
+    }
+}
+/**
+ * Configuration file not found.
+ */
+export class ConfigNotFoundError extends ConfigError {
+    /** Path that was searched */
+    path;
+    constructor(path, context) {
+        super(`Configuration file not found: ${path}`, {
+            ...context,
+            metadata: { ...context?.metadata, path },
+        });
+        this.name = 'ConfigNotFoundError';
+        this.path = path;
+    }
+}
+/**
+ * Configuration validation failed.
+ */
+export class ConfigValidationError extends ConfigError {
+    /** Validation errors */
+    validationErrors;
+    constructor(errors, context) {
+        super(`Configuration validation failed:\n${errors.map((e) => `  - ${e}`).join('\n')}`, {
+            ...context,
+            metadata: { ...context?.metadata, validationErrors: errors },
+        });
+        this.name = 'ConfigValidationError';
+        this.validationErrors = errors;
+    }
+}
+/**
+ * Check if an error is a BellwetherError.
+ */
+export function isBellwetherError(error) {
+    return error instanceof BellwetherError;
+}
+/**
+ * Check if an error is retryable.
+ */
+export function isRetryable(error) {
+    if (isBellwetherError(error)) {
+        return error.retryable === 'retryable';
+    }
+    // For unknown errors, default to retryable for transient issues
+    if (error instanceof Error) {
+        const message = error.message.toLowerCase();
+        return (message.includes('timeout') ||
+            message.includes('econnreset') ||
+            message.includes('econnrefused') ||
+            message.includes('rate limit') ||
+            message.includes('429'));
+    }
+    return false;
+}
+/**
+ * Wrap an unknown error in a BellwetherError.
+ */
+export function wrapError(error, context) {
+    if (isBellwetherError(error)) {
+        return context ? error.withContext(context) : error;
+    }
+    const originalError = error instanceof Error ? error : new Error(String(error));
+    return new BellwetherError(originalError.message, {
+        code: 'UNKNOWN_ERROR',
+        severity: 'medium',
+        retryable: 'unknown',
+        context,
+        cause: originalError,
+    });
+}
+/**
+ * Extract error message from any error type.
+ */
+export function getErrorMessage(error) {
+    if (isBellwetherError(error)) {
+        return error.message;
+    }
+    if (error instanceof Error) {
+        return error.message;
+    }
+    return String(error);
+}
+/**
+ * Create timing context for error tracking.
+ */
+export function createTimingContext(startedAt) {
+    const failedAt = new Date();
+    return {
+        startedAt,
+        failedAt,
+        durationMs: failedAt.getTime() - startedAt.getTime(),
+    };
+}
+//# sourceMappingURL=types.js.map

package/dist/index.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Bellwether - Interview MCP servers to generate behavioral documentation
+ *
+ * @packageDocumentation
+ */
+export { MCPClient, type MCPClientOptions } from './transport/mcp-client.js';
+export { BaseTransport, type BaseTransportConfig, type TransportType } from './transport/base-transport.js';
+export { StdioTransport, type StdioTransportConfig } from './transport/stdio-transport.js';
+export { SSETransport, type SSETransportConfig } from './transport/sse-transport.js';
+export { HTTPTransport, type HTTPTransportConfig } from './transport/http-transport.js';
+export type { JSONRPCRequest, JSONRPCResponse, JSONRPCError, JSONRPCMessage, MCPTool, MCPPrompt, MCPResource, MCPServerCapabilities, MCPServerInfo, MCPInitializeResult, MCPToolCallResult, MCPResourceReadResult, MCPContentBlock, } from './transport/types.js';
+export { discover, parseToolDetail, summarizeDiscovery } from './discovery/discovery.js';
+export type { DiscoveryResult, ToolDetail, ToolInputSchema } from './discovery/types.js';
+export { Interviewer, DEFAULT_CONFIG } from './interview/interviewer.js';
+export { Orchestrator } from './interview/orchestrator.js';
+export type { InterviewConfig, InterviewResult, InterviewQuestion, ToolProfile, ToolInteraction, InterviewMetadata, PromptProfile, PromptInteraction, ResourceProfile, ResourceInteraction, ResourceQuestion, } from './interview/types.js';
+export type { LLMClient, Message, CompletionOptions } from './llm/client.js';
+export { OpenAIClient, type OpenAIClientOptions } from './llm/openai.js';
+export { generateAgentsMd, generateContractMd, generateJsonReport } from './docs/generator.js';
+export { loadConfig, ConfigNotFoundError } from './config/loader.js';
+export type { BellwetherConfig } from './config/loader.js';
+export { validateConfig, findConfigFile } from './config/validator.js';
+export { generateConfigTemplate, generatePresetConfig, PRESETS } from './config/template.js';
+export { createLogger, getLogger, configureLogger, resetLogger, childLogger, startTiming, LOG_LEVEL_VALUES, isLevelEnabled, } from './logging/logger.js';
+export type { LogLevel, LoggerConfig, Logger, TimingResult } from './logging/logger.js';
+export { loadScenariosFromFile, tryLoadDefaultScenarios, generateSampleScenariosYaml, DEFAULT_SCENARIOS_FILE, evaluateAssertion, evaluateAssertions, getValueAtPath, formatAssertionResults, } from './scenarios/index.js';
+export type { TestScenario, PromptScenario, ScenarioAssertion, AssertionCondition, LoadedScenarios, AssertionResult, ScenarioResult, } from './scenarios/index.js';
+export { RegistryClient, REGISTRY_BASE_URL, API_VERSION, generateRunCommand, generatePackageCommand, formatServerEntry, formatServerList, } from './registry/index.js';
+export type { RegistryRepository, RegistryPackageArgument, RegistryTransport, RegistryPackage, RegistryServerMeta, RegistryServer, RegistryServerEntry, RegistryResponseMetadata, RegistryServersResponse, ListServersOptions, } from './registry/index.js';
+export { generateVerificationResult, generateVerificationReport, generateVerificationBadge, generateBadgeUrl, generateBadgeMarkdown, isVerificationValid, } from './verification/index.js';
+export type { VerificationStatus, VerificationTier, VerificationResult, VerificationBadge, VerificationConfig, VerificationReport, VerificationSubmission, } from './verification/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,32 @@
+/**
+ * Bellwether - Interview MCP servers to generate behavioral documentation
+ *
+ * @packageDocumentation
+ */
+// Transport layer
+export { MCPClient } from './transport/mcp-client.js';
+export { BaseTransport } from './transport/base-transport.js';
+export { StdioTransport } from './transport/stdio-transport.js';
+export { SSETransport } from './transport/sse-transport.js';
+export { HTTPTransport } from './transport/http-transport.js';
+// Discovery
+export { discover, parseToolDetail, summarizeDiscovery } from './discovery/discovery.js';
+// Interview
+export { Interviewer, DEFAULT_CONFIG } from './interview/interviewer.js';
+export { Orchestrator } from './interview/orchestrator.js';
+export { OpenAIClient } from './llm/openai.js';
+// Documentation
+export { generateAgentsMd, generateContractMd, generateJsonReport } from './docs/generator.js';
+// Config
+export { loadConfig, ConfigNotFoundError } from './config/loader.js';
+export { validateConfig, findConfigFile } from './config/validator.js';
+export { generateConfigTemplate, generatePresetConfig, PRESETS } from './config/template.js';
+// Logging
+export { createLogger, getLogger, configureLogger, resetLogger, childLogger, startTiming, LOG_LEVEL_VALUES, isLevelEnabled, } from './logging/logger.js';
+// Scenarios
+export { loadScenariosFromFile, tryLoadDefaultScenarios, generateSampleScenariosYaml, DEFAULT_SCENARIOS_FILE, evaluateAssertion, evaluateAssertions, getValueAtPath, formatAssertionResults, } from './scenarios/index.js';
+// Registry
+export { RegistryClient, REGISTRY_BASE_URL, API_VERSION, generateRunCommand, generatePackageCommand, formatServerEntry, formatServerList, } from './registry/index.js';
+// Verification
+export { generateVerificationResult, generateVerificationReport, generateVerificationBadge, generateBadgeUrl, generateBadgeMarkdown, isVerificationValid, } from './verification/index.js';
+//# sourceMappingURL=index.js.map

package/dist/interview/dependency-resolver.d.ts

@@ -0,0 +1,11 @@
+import type { MCPTool } from '../transport/types.js';
+import type { ToolDependencyInfo } from './types.js';
+/**
+ * Resolve tool dependencies from schema analysis.
+ */
+export declare function resolveToolDependencies(tools: MCPTool[]): ToolDependencyInfo[];
+/**
+ * Get tools ordered by dependency sequence.
+ */
+export declare function getDependencyOrder(dependencies: ToolDependencyInfo[]): string[];
+//# sourceMappingURL=dependency-resolver.d.ts.map

package/dist/interview/dependency-resolver.js

@@ -0,0 +1,32 @@
+import { analyzeDependencies } from '../baseline/dependency-analyzer.js';
+/**
+ * Resolve tool dependencies from schema analysis.
+ */
+export function resolveToolDependencies(tools) {
+    const graph = analyzeDependencies(tools);
+    const layerMap = new Map();
+    graph.layers.forEach((layer, index) => {
+        for (const toolName of layer) {
+            layerMap.set(toolName, index);
+        }
+    });
+    return tools.map((tool) => {
+        const dependsOn = graph.edges.filter((e) => e.to === tool.name).map((e) => e.from);
+        const provides = graph.edges.filter((e) => e.from === tool.name).map((e) => e.to);
+        return {
+            tool: tool.name,
+            dependsOn: Array.from(new Set(dependsOn)),
+            providesOutputFor: Array.from(new Set(provides)),
+            sequencePosition: layerMap.get(tool.name) ?? 0,
+        };
+    });
+}
+/**
+ * Get tools ordered by dependency sequence.
+ */
+export function getDependencyOrder(dependencies) {
+    return [...dependencies]
+        .sort((a, b) => a.sequencePosition - b.sequencePosition || a.tool.localeCompare(b.tool))
+        .map((d) => d.tool);
+}
+//# sourceMappingURL=dependency-resolver.js.map