@dotsetlabs/bellwether 0.10.0

package/dist/interview/interviewer.d.ts

@@ -0,0 +1,232 @@
+import type { MCPClient } from '../transport/mcp-client.js';
+import type { DiscoveryResult } from '../discovery/types.js';
+import type { LLMClient } from '../llm/client.js';
+import type { InterviewConfig, InterviewResult, ServerContext } from './types.js';
+import type { Persona } from '../persona/types.js';
+import type { TestScenario, PromptScenario, ScenarioResult } from '../scenarios/types.js';
+/**
+ * Default interview configuration.
+ */
+export declare const DEFAULT_CONFIG: InterviewConfig;
+/**
+ * Default personas to use if none specified.
+ * Uses Technical Writer only for a fast, cost-effective default experience.
+ * Use --security or --personas to add more personas.
+ */
+export declare const DEFAULT_PERSONAS: Persona[];
+export interface InterviewProgress {
+    phase: 'starting' | 'interviewing' | 'prompts' | 'resources' | 'workflows' | 'synthesizing' | 'complete';
+    currentTool?: string;
+    currentPersona?: string;
+    personasCompleted: number;
+    totalPersonas: number;
+    toolsCompleted: number;
+    totalTools: number;
+    questionsAsked: number;
+    /** Summary for the last completed tool (check mode) */
+    lastCompletedTool?: ToolProgressSummary;
+    /** Current workflow being executed */
+    currentWorkflow?: string;
+    /** Number of workflows completed */
+    workflowsCompleted?: number;
+    /** Total workflows to execute */
+    totalWorkflows?: number;
+    /** Number of prompts completed */
+    promptsCompleted?: number;
+    /** Total prompts to interview */
+    totalPrompts?: number;
+    /** Number of resources completed */
+    resourcesCompleted?: number;
+    /** Total resources to interview */
+    totalResources?: number;
+}
+export interface ToolProgressSummary {
+    toolName: string;
+    totalTests: number;
+    passedTests: number;
+    validationTotal: number;
+    validationPassed: number;
+    avgMs: number;
+    skipped?: boolean;
+    skipReason?: string;
+    mocked?: boolean;
+    mockService?: string;
+}
+export type ProgressCallback = (progress: InterviewProgress) => void;
+/**
+ * Interviewer conducts the interview process using the orchestrator.
+ * Supports streaming output for real-time feedback during LLM operations.
+ * Supports parallel persona execution for improved performance.
+ * Supports caching tool responses and LLM analysis for efficiency.
+ *
+ * Two modes of operation:
+ * - Check mode: No LLM required, uses fallback questions and simple analysis
+ * - Explore mode: LLM required for question generation and behavioral analysis
+ */
+export declare class Interviewer {
+    private llm;
+    private config;
+    private personas;
+    private logger;
+    private serverContext?;
+    private cache?;
+    private rateLimiter?;
+    private responseSchemas;
+    private rateLimitEvents;
+    private rateLimitRetries;
+    private externalServiceStatuses;
+    private skippedTools;
+    private mockedTools;
+    /**
+     * Create an Interviewer for explore mode (LLM-powered behavioral analysis).
+     *
+     * @param llm - LLM client for question generation and analysis
+     * @param config - Interview configuration
+     */
+    constructor(llm: LLMClient, config?: Partial<InterviewConfig>);
+    /**
+     * Create an Interviewer for check mode (no LLM, deterministic).
+     *
+     * @param llm - null for check mode
+     * @param config - Interview configuration (must have checkMode: true)
+     */
+    constructor(llm: null, config: Partial<InterviewConfig> & {
+        checkMode: true;
+    });
+    /**
+     * Create an orchestrator with streaming and caching enabled if configured.
+     * Throws an error if called in check mode since orchestrator requires LLM.
+     */
+    private createOrchestrator;
+    /**
+     * Generate simple analysis for check/fast mode.
+     * Avoids LLM calls by providing basic success/error messages.
+     */
+    private generateSimpleAnalysis;
+    /**
+     * Assess whether the tool interaction outcome matched expectations.
+     */
+    private assessOutcome;
+    /**
+     * Infer expected outcome when not explicitly provided.
+     */
+    private inferExpectedOutcome;
+    private extractErrorMessage;
+    private resolveExternalServiceDecision;
+    private recordRateLimitEvent;
+    private callToolWithPolicies;
+    /**
+     * Check if we're in fast/check mode (no LLM calls).
+     */
+    private isCheckMode;
+    /**
+     * Extract server context by probing discovery tools.
+     * Looks for tools like list_allowed_directories to understand constraints.
+     */
+    extractServerContext(client: MCPClient, discovery: DiscoveryResult): Promise<ServerContext>;
+    /**
+     * Parse allowed directories from tool response text.
+     */
+    private parseAllowedDirectories;
+    /**
+     * Set server context directly (e.g., from CLI arguments).
+     */
+    setServerContext(context: ServerContext): void;
+    /**
+     * Run a complete interview on a connected MCP server.
+     * Supports multiple personas - runs each persona's interview and aggregates findings.
+     */
+    interview(client: MCPClient, discovery: DiscoveryResult, onProgress?: ProgressCallback): Promise<InterviewResult>;
+    /**
+     * Classify errors from interactions to separate tool correctness from environment issues.
+     */
+    private classifyErrors;
+    /**
+     * Aggregate findings from multiple personas into a single tool profile.
+     */
+    private aggregateFindings;
+    /**
+     * Execute a tool call with retry logic for recoverable errors.
+     * Learns from errors and can update server context based on error messages.
+     * Uses caching to avoid redundant tool calls with identical arguments.
+     */
+    private executeWithRetry;
+    /**
+     * Learn server constraints from error messages.
+     * Updates server context with discovered restrictions.
+     */
+    private learnFromError;
+    /**
+     * Interview all tools with a single persona.
+     * Designed for parallel execution across personas.
+     *
+     * @param client - MCP client for tool calls
+     * @param discovery - Discovery result with available tools
+     * @param persona - Persona to use for this interview
+     * @param toolCallMutex - Mutex for serializing tool calls (shared resource)
+     * @returns PersonaInterviewData with all interactions and findings
+     */
+    private interviewPersona;
+    /**
+     * Aggregate results from parallel persona interviews.
+     */
+    private aggregateParallelResults;
+    /**
+     * Interview a single tool in check mode (parallel-safe).
+     * Designed for parallel tool testing with minimal overhead.
+     *
+     * @param client - MCP client for tool calls
+     * @param tool - Tool to test
+     * @param toolCallMutex - Mutex for serializing tool calls (shared resource)
+     * @returns ToolCheckResult with interactions and stats
+     */
+    private interviewToolInCheckMode;
+    /**
+     * Get fallback questions for a tool without requiring an orchestrator.
+     * Used in check mode when parallel tool testing is enabled.
+     *
+     * Uses the SchemaTestGenerator to produce comprehensive deterministic tests
+     * including boundaries, type coercion, enum validation, and error handling.
+     */
+    private getFallbackQuestionsForTool;
+    /**
+     * Run parallel tool testing in check mode.
+     * Tests all tools concurrently with a configurable worker limit.
+     *
+     * @param client - MCP client for tool calls
+     * @param tools - Tools to test
+     * @param onProgress - Progress callback
+     * @returns Aggregated tool profiles
+     */
+    private interviewToolsInParallel;
+    private buildToolProgressSummary;
+    /**
+     * Convert a TestScenario to an InterviewQuestion.
+     */
+    private scenarioToQuestion;
+    /**
+     * Get custom scenarios for a specific tool.
+     */
+    private getScenariosForTool;
+    /**
+     * Get custom scenarios for a specific prompt.
+     */
+    private getScenariosForPrompt;
+    /**
+     * Execute custom test scenarios for a tool.
+     * Returns scenario results with assertion evaluations.
+     */
+    executeToolScenarios(client: MCPClient, toolName: string, scenarios: TestScenario[]): Promise<ScenarioResult[]>;
+    /**
+     * Execute custom test scenarios for a prompt.
+     * Returns scenario results with assertion evaluations.
+     */
+    executePromptScenarios(client: MCPClient, promptName: string, scenarios: PromptScenario[]): Promise<ScenarioResult[]>;
+    /**
+     * Execute workflow discovery and/or execution.
+     * Discovers workflows using LLM if enabled, loads from file if provided,
+     * and executes all workflows against the MCP server.
+     */
+    private executeWorkflows;
+}
+//# sourceMappingURL=interviewer.d.ts.map