@dotsetlabs/bellwether 0.10.0

package/dist/interview/mock-response-generator.d.ts

@@ -0,0 +1,7 @@
+import type { MCPTool, MCPToolCallResult } from '../transport/types.js';
+import type { ExternalServiceName } from '../baseline/external-dependency-detector.js';
+/**
+ * Generate a deterministic mock response for an external service tool.
+ */
+export declare function generateMockResponse(tool: MCPTool, serviceName: ExternalServiceName): MCPToolCallResult;
+//# sourceMappingURL=mock-response-generator.d.ts.map

package/dist/interview/mock-response-generator.js

@@ -0,0 +1,102 @@
+/**
+ * Generate a deterministic mock response for an external service tool.
+ */
+export function generateMockResponse(tool, serviceName) {
+    const payload = buildMockPayload(tool, serviceName);
+    return {
+        content: [
+            {
+                type: 'text',
+                text: JSON.stringify(payload, null, 2),
+            },
+        ],
+        isError: false,
+    };
+}
+function buildMockPayload(tool, serviceName) {
+    const base = {
+        mock: true,
+        service: serviceName,
+        tool: tool.name,
+        generatedAt: new Date().toISOString(),
+    };
+    switch (serviceName) {
+        case 'plaid':
+            return {
+                ...base,
+                requestId: 'req_plaid_mock_001',
+                item: { item_id: 'item_mock_123', access_token: 'access-mock-token' },
+                accounts: [
+                    { account_id: 'acc_mock_1', name: 'Checking', type: 'depository', balances: { current: 1200.55 } },
+                ],
+            };
+        case 'stripe':
+            return {
+                ...base,
+                id: 'ch_mock_123',
+                status: 'succeeded',
+                amount: 2000,
+                currency: 'usd',
+            };
+        case 'aws':
+            return {
+                ...base,
+                requestId: 'aws_mock_request',
+                result: { bucket: 'mock-bucket', key: 'mock-key', status: 'ok' },
+            };
+        case 'openai':
+            return {
+                ...base,
+                id: 'chatcmpl_mock_123',
+                model: 'gpt-mock',
+                choices: [{ index: 0, message: { role: 'assistant', content: 'Mock response.' } }],
+            };
+        case 'anthropic':
+            return {
+                ...base,
+                id: 'msg_mock_123',
+                model: 'claude-mock',
+                content: [{ type: 'text', text: 'Mock response.' }],
+            };
+        case 'firebase':
+            return {
+                ...base,
+                id: 'doc_mock_123',
+                status: 'ok',
+                data: {},
+            };
+        case 'twilio':
+            return {
+                ...base,
+                sid: 'SMmock123',
+                status: 'queued',
+                to: '+15551234567',
+            };
+        case 'sendgrid':
+            return {
+                ...base,
+                message: 'Mock email accepted',
+                status: 'queued',
+            };
+        case 'github':
+            return {
+                ...base,
+                id: 123456,
+                status: 'ok',
+                url: 'https://api.github.com/mock',
+            };
+        case 'database':
+            return {
+                ...base,
+                status: 'ok',
+                rows: [],
+            };
+        default:
+            return {
+                ...base,
+                status: 'ok',
+                data: {},
+            };
+    }
+}
+//# sourceMappingURL=mock-response-generator.js.map

package/dist/interview/orchestrator.d.ts

@@ -0,0 +1,237 @@
+import type { LLMClient } from '../llm/client.js';
+import type { MCPTool, MCPToolCallResult, MCPPrompt, MCPPromptGetResult, MCPResource, MCPResourceReadResult } from '../transport/types.js';
+import type { InterviewQuestion, ToolProfile, ServerContext, PromptQuestion, PromptProfile, ResourceQuestion, ResourceProfile } from './types.js';
+import type { DiscoveryResult } from '../discovery/types.js';
+import type { Persona } from '../persona/types.js';
+import type { ToolResponseCache } from '../cache/response-cache.js';
+/**
+ * Streaming callback for orchestrator operations.
+ */
+export interface OrchestratorStreamingCallbacks {
+    /** Called when streaming starts for an operation */
+    onStart?: (operation: string, context?: string) => void;
+    /** Called with each chunk of streaming text */
+    onChunk?: (chunk: string, operation: string) => void;
+    /** Called when streaming completes for an operation */
+    onComplete?: (text: string, operation: string) => void;
+    /** Called if an error occurs during streaming */
+    onError?: (error: Error, operation: string) => void;
+}
+/**
+ * Orchestrator uses an LLM to generate interview questions and synthesize findings.
+ * Optionally accepts a Persona to customize the interview style.
+ * Supports streaming output for real-time feedback during LLM operations.
+ */
+export declare class Orchestrator {
+    private llm;
+    private persona;
+    private serverContext?;
+    private logger;
+    private streamingCallbacks?;
+    private useStreaming;
+    private cache?;
+    constructor(llm: LLMClient, persona?: Persona, serverContext?: ServerContext, cache?: ToolResponseCache);
+    /**
+     * Enable streaming with callbacks.
+     */
+    enableStreaming(callbacks: OrchestratorStreamingCallbacks): void;
+    /**
+     * Disable streaming.
+     */
+    disableStreaming(): void;
+    /**
+     * Check if streaming is enabled.
+     */
+    isStreamingEnabled(): boolean;
+    /**
+     * Create streaming options for an LLM call.
+     */
+    private createStreamingOptions;
+    /**
+     * Complete an LLM call, using streaming if enabled.
+     * Falls back to non-streaming if streaming returns empty content.
+     */
+    private completeWithStreaming;
+    /**
+     * Set server context for contextually appropriate question generation.
+     */
+    setServerContext(context: ServerContext): void;
+    /**
+     * Get the current server context.
+     */
+    getServerContext(): ServerContext | undefined;
+    /**
+     * Get the current persona.
+     */
+    getPersona(): Persona;
+    /**
+     * Set a new persona for subsequent operations.
+     */
+    setPersona(persona: Persona): void;
+    /**
+     * Get the system prompt, combining persona prompt with additional context.
+     */
+    private getSystemPrompt;
+    /**
+     * Get categories to focus on based on persona bias.
+     */
+    private getCategoryDistribution;
+    /**
+     * Generate interview questions for a tool.
+     * Optionally accepts previous errors to learn from and avoid.
+     *
+     * Error handling strategy:
+     * - Refusals: Use fallback questions (no retry)
+     * - Rate limits: Retry with backoff
+     * - Timeouts: Retry once, then fallback
+     * - Auth errors: Log and use fallback (no retry)
+     * - Network errors: Retry with backoff
+     * - Unknown errors: Log and use fallback
+     */
+    generateQuestions(tool: MCPTool, maxQuestions?: number, skipErrorTests?: boolean, previousErrors?: Array<{
+        args: Record<string, unknown>;
+        error: string;
+    }>): Promise<InterviewQuestion[]>;
+    /**
+     * Analyze a tool response and generate behavioral notes.
+     * Uses cache to avoid redundant LLM calls for identical tool responses.
+     */
+    analyzeResponse(tool: MCPTool, question: InterviewQuestion, response: MCPToolCallResult | null, error: string | null): Promise<string>;
+    /**
+     * Synthesize findings for a single tool into a profile.
+     */
+    synthesizeToolProfile(tool: MCPTool, interactions: {
+        question: InterviewQuestion;
+        response: MCPToolCallResult | null;
+        error: string | null;
+        analysis: string;
+    }[]): Promise<Omit<ToolProfile, 'interactions'>>;
+    /**
+     * Generate overall summary for the interview result.
+     */
+    synthesizeOverall(discovery: DiscoveryResult, toolProfiles: ToolProfile[]): Promise<{
+        summary: string;
+        limitations: string[];
+        recommendations: string[];
+    }>;
+    /**
+     * Get fallback questions without LLM call (for fast CI mode).
+     * Enhanced to generate comprehensive test cases from schema analysis.
+     */
+    getFallbackQuestions(tool: MCPTool, skipErrorTests: boolean): InterviewQuestion[];
+    /**
+     * Generate comprehensive test cases for check mode.
+     * Analyzes schema to create meaningful tests without LLM.
+     */
+    private generateStructuralTestCases;
+    /**
+     * Build args from property-level default values.
+     * Also includes required parameters with smart defaults.
+     */
+    private buildArgsFromDefaults;
+    /**
+     * Build args from property-level example values.
+     * Also includes required parameters with smart defaults.
+     */
+    private buildArgsFromExamples;
+    /**
+     * Build smart default args based on parameter analysis.
+     */
+    private buildSmartDefaultArgs;
+    /**
+     * Generate a smart value for a parameter based on comprehensive schema analysis.
+     * @param depth - Current recursion depth for circular schema protection
+     */
+    private generateSmartValue;
+    /**
+     * Generate a smart string value based on format, pattern, and name hints.
+     * Works with or without a schema - when schema is absent, uses name-based inference only.
+     */
+    private generateSmartString;
+    /**
+     * Generate a smart number value respecting constraints.
+     */
+    private generateSmartNumber;
+    /**
+     * Generate a smart array value.
+     * @param depth - Current recursion depth for circular schema protection
+     */
+    private generateSmartArray;
+    /**
+     * Generate a smart object value.
+     * @param depth - Current recursion depth for circular schema protection
+     */
+    private generateSmartObject;
+    /**
+     * Infer value from parameter name when no type info available.
+     */
+    private inferValueFromName;
+    /**
+     * Get the primary type from a schema type definition.
+     * Handles both single type strings and type arrays (e.g., ['string', 'null']).
+     */
+    private getSchemaType;
+    /**
+     * Generate test cases for enum parameters.
+     */
+    private generateEnumTests;
+    /**
+     * Generate boundary tests for numeric parameters.
+     */
+    private generateBoundaryTests;
+    /**
+     * Generate tests for optional parameters.
+     */
+    private generateOptionalParamTests;
+    /**
+     * Generate tests with invalid types to check error handling.
+     */
+    private generateInvalidTypeTests;
+    /**
+     * Generate interview questions for an MCP prompt.
+     */
+    generatePromptQuestions(prompt: MCPPrompt, maxQuestions?: number): Promise<PromptQuestion[]>;
+    /**
+     * Analyze a prompt response.
+     */
+    analyzePromptResponse(prompt: MCPPrompt, question: PromptQuestion, response: MCPPromptGetResult | null, error: string | null): Promise<string>;
+    /**
+     * Synthesize findings for a prompt into a profile.
+     */
+    synthesizePromptProfile(prompt: MCPPrompt, interactions: Array<{
+        question: PromptQuestion;
+        response: MCPPromptGetResult | null;
+        error: string | null;
+        analysis: string;
+    }>): Promise<Omit<PromptProfile, 'interactions'>>;
+    /**
+     * Fallback questions when LLM fails for prompts.
+     */
+    private generateFallbackPromptQuestions;
+    /**
+     * Generate interview questions for an MCP resource.
+     */
+    generateResourceQuestions(resource: MCPResource, maxQuestions?: number): Promise<ResourceQuestion[]>;
+    /**
+     * Analyze a resource read response.
+     */
+    analyzeResourceResponse(resource: MCPResource, question: ResourceQuestion, response: MCPResourceReadResult | null, error: string | null): Promise<string>;
+    /**
+     * Synthesize findings for a resource into a profile.
+     */
+    synthesizeResourceProfile(resource: MCPResource, interactions: Array<{
+        question: ResourceQuestion;
+        response: MCPResourceReadResult | null;
+        error: string | null;
+        analysis: string;
+    }>): Promise<Omit<ResourceProfile, 'interactions' | 'contentPreview'>>;
+    /**
+     * Fallback questions when LLM fails for resources.
+     */
+    private generateFallbackResourceQuestions;
+    /**
+     * Summarize resource content for analysis prompts.
+     */
+    private summarizeResourceContent;
+}
+//# sourceMappingURL=orchestrator.d.ts.map