@dotsetlabs/bellwether 0.10.0

package/dist/registry/types.d.ts

@@ -0,0 +1,140 @@
+/**
+ * MCP Registry API types.
+ * Based on the official MCP Registry at registry.modelcontextprotocol.io
+ */
+/**
+ * Repository information for a server.
+ */
+export interface RegistryRepository {
+    /** Repository URL (e.g., GitHub URL) */
+    url: string;
+    /** Source type (e.g., "github") */
+    source: string;
+    /** Subfolder within the repository, if applicable */
+    subfolder?: string;
+}
+/**
+ * Package argument definition.
+ */
+export interface RegistryPackageArgument {
+    /** Argument name */
+    name: string;
+    /** Description of the argument */
+    description?: string;
+    /** Argument type ("named" or "positional") */
+    type: 'named' | 'positional';
+    /** Value format */
+    format?: 'string' | 'number' | 'boolean';
+    /** Whether the argument is required */
+    isRequired?: boolean;
+    /** Default value */
+    default?: string | number | boolean;
+}
+/**
+ * Transport configuration for a package.
+ */
+export interface RegistryTransport {
+    /** Transport type */
+    type: 'stdio' | 'sse' | 'streamable-http';
+    /** URL for remote transports */
+    url?: string;
+}
+/**
+ * Package distribution information.
+ */
+export interface RegistryPackage {
+    /** Package registry type (npm, pip, cargo, etc.) */
+    registryType: string;
+    /** Base URL of the registry */
+    registryBaseUrl?: string;
+    /** Package identifier/name */
+    identifier: string;
+    /** Package version */
+    version?: string;
+    /** Transport configuration */
+    transport?: RegistryTransport;
+    /** Package arguments */
+    packageArguments?: RegistryPackageArgument[];
+    /** Runtime (e.g., "node", "python") */
+    runtime?: string;
+}
+/**
+ * Server metadata from the registry.
+ */
+export interface RegistryServerMeta {
+    /** Server status (e.g., "active") */
+    status?: string;
+    /** When the server was published */
+    publishedAt?: string;
+    /** When the server was last updated */
+    updatedAt?: string;
+    /** Whether this is the latest version */
+    isLatest?: boolean;
+}
+/**
+ * Server definition from the registry.
+ */
+export interface RegistryServer {
+    /** JSON Schema reference */
+    $schema?: string;
+    /** Server name (namespace/name format) */
+    name: string;
+    /** Short description */
+    description?: string;
+    /** Repository information */
+    repository?: RegistryRepository;
+    /** Server version */
+    version?: string;
+    /** Available packages/distributions */
+    packages?: RegistryPackage[];
+    /** Icon URL */
+    iconUrl?: string;
+    /** Homepage URL */
+    homepage?: string;
+    /** License */
+    license?: string;
+    /** Keywords/tags */
+    keywords?: string[];
+}
+/**
+ * Server entry in registry response.
+ */
+export interface RegistryServerEntry {
+    /** Server definition */
+    server: RegistryServer;
+    /** Registry metadata */
+    _meta?: {
+        'io.modelcontextprotocol.registry/official'?: RegistryServerMeta;
+        [key: string]: RegistryServerMeta | undefined;
+    };
+}
+/**
+ * Response metadata.
+ */
+export interface RegistryResponseMetadata {
+    /** Total count of results */
+    count: number;
+    /** Cursor for pagination */
+    nextCursor?: string;
+}
+/**
+ * Response from the servers list endpoint.
+ */
+export interface RegistryServersResponse {
+    /** List of servers */
+    servers: RegistryServerEntry[];
+    /** Response metadata */
+    metadata: RegistryResponseMetadata;
+}
+/**
+ * Options for listing servers.
+ */
+export interface ListServersOptions {
+    /** Search query */
+    search?: string;
+    /** Maximum number of results */
+    limit?: number;
+    /** Pagination cursor */
+    cursor?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/registry/types.js

@@ -0,0 +1,6 @@
+/**
+ * MCP Registry API types.
+ * Based on the official MCP Registry at registry.modelcontextprotocol.io
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/scenarios/evaluator.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Scenario assertion evaluator.
+ *
+ * Evaluates assertions against tool/prompt responses.
+ */
+import type { ScenarioAssertion, AssertionResult } from './types.js';
+import { getValueAtPath } from '../utils/jsonpath.js';
+export { getValueAtPath };
+/**
+ * Extract and parse the actual response content from an MCP tool call result.
+ *
+ * MCP returns responses in this format:
+ * {
+ *   content: [{ type: 'text', text: '{"success": true, ...}' }],
+ *   isError: false
+ * }
+ *
+ * This function extracts the JSON from content[0].text and parses it,
+ * allowing assertions to check paths like 'success' and 'note.id' directly.
+ *
+ * For error responses (isError: true), returns an object with:
+ * { error: true, message: "...", isError: true }
+ *
+ * Falls back to returning the original response if:
+ * - Response is not an MCP tool call result
+ * - No text content found
+ * - Text is not valid JSON (for success responses)
+ */
+export declare function extractResponseContent(response: unknown): unknown;
+/**
+ * Evaluate a single assertion against a response.
+ */
+export declare function evaluateAssertion(assertion: ScenarioAssertion, response: unknown, isError: boolean): AssertionResult;
+/**
+ * Evaluate all assertions for a response.
+ * Extracts JSON content from MCP results for direct path access.
+ */
+export declare function evaluateAssertions(assertions: ScenarioAssertion[], response: unknown, isError: boolean): AssertionResult[];
+/**
+ * Format assertion results for display.
+ */
+export declare function formatAssertionResults(results: AssertionResult[]): string;
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/scenarios/evaluator.js

@@ -0,0 +1,206 @@
+/**
+ * Scenario assertion evaluator.
+ *
+ * Evaluates assertions against tool/prompt responses.
+ */
+import { getValueAtPath } from '../utils/jsonpath.js';
+// Re-export for backwards compatibility
+export { getValueAtPath };
+/**
+ * Extract and parse the actual response content from an MCP tool call result.
+ *
+ * MCP returns responses in this format:
+ * {
+ *   content: [{ type: 'text', text: '{"success": true, ...}' }],
+ *   isError: false
+ * }
+ *
+ * This function extracts the JSON from content[0].text and parses it,
+ * allowing assertions to check paths like 'success' and 'note.id' directly.
+ *
+ * For error responses (isError: true), returns an object with:
+ * { error: true, message: "...", isError: true }
+ *
+ * Falls back to returning the original response if:
+ * - Response is not an MCP tool call result
+ * - No text content found
+ * - Text is not valid JSON (for success responses)
+ */
+export function extractResponseContent(response) {
+    // Handle null/undefined
+    if (response === null || response === undefined) {
+        return response;
+    }
+    // Check if this looks like an MCP tool call result
+    if (typeof response === 'object' && 'content' in response) {
+        const mcpResult = response;
+        // Find text content
+        const textContent = mcpResult.content?.find((c) => c.type === 'text' && c.text);
+        // If this is an error response, wrap the message in an error object
+        // This allows assertions like { path: 'error', condition: 'exists' } to work
+        if (mcpResult.isError) {
+            const errorMessage = textContent?.text ?? 'Unknown error';
+            return {
+                error: true,
+                message: errorMessage,
+                isError: true,
+            };
+        }
+        // For success responses, try to parse JSON
+        if (textContent?.text) {
+            try {
+                // Try to parse as JSON
+                const parsed = JSON.parse(textContent.text);
+                return parsed;
+            }
+            catch {
+                // Not valid JSON, return the raw text
+                return textContent.text;
+            }
+        }
+    }
+    // Not an MCP response, return as-is
+    return response;
+}
+/**
+ * Evaluate a single assertion against a response.
+ */
+export function evaluateAssertion(assertion, response, isError) {
+    const actualValue = getValueAtPath(response, assertion.path);
+    try {
+        switch (assertion.condition) {
+            case 'exists':
+                return {
+                    assertion,
+                    passed: actualValue !== undefined,
+                    actualValue,
+                    error: actualValue === undefined
+                        ? assertion.message ?? `Expected path "${assertion.path}" to exist`
+                        : undefined,
+                };
+            case 'equals': {
+                const isEqual = JSON.stringify(actualValue) === JSON.stringify(assertion.value);
+                return {
+                    assertion,
+                    passed: isEqual,
+                    actualValue,
+                    error: isEqual
+                        ? undefined
+                        : assertion.message ??
+                            `Expected ${JSON.stringify(actualValue)} to equal ${JSON.stringify(assertion.value)}`,
+                };
+            }
+            case 'contains': {
+                let containsValue = false;
+                if (typeof actualValue === 'string' && typeof assertion.value === 'string') {
+                    containsValue = actualValue.includes(assertion.value);
+                }
+                else if (Array.isArray(actualValue)) {
+                    containsValue = actualValue.some((item) => JSON.stringify(item) === JSON.stringify(assertion.value));
+                }
+                return {
+                    assertion,
+                    passed: containsValue,
+                    actualValue,
+                    error: containsValue
+                        ? undefined
+                        : assertion.message ??
+                            `Expected ${JSON.stringify(actualValue)} to contain ${JSON.stringify(assertion.value)}`,
+                };
+            }
+            case 'truthy': {
+                const isTruthy = Boolean(actualValue);
+                return {
+                    assertion,
+                    passed: isTruthy,
+                    actualValue,
+                    error: isTruthy
+                        ? undefined
+                        : assertion.message ?? `Expected path "${assertion.path}" to be truthy`,
+                };
+            }
+            case 'type': {
+                let actualType;
+                if (Array.isArray(actualValue)) {
+                    actualType = 'array';
+                }
+                else if (actualValue === null) {
+                    actualType = 'null';
+                }
+                else {
+                    actualType = typeof actualValue;
+                }
+                const typeMatches = actualType === assertion.value;
+                return {
+                    assertion,
+                    passed: typeMatches,
+                    actualValue,
+                    error: typeMatches
+                        ? undefined
+                        : assertion.message ??
+                            `Expected type "${assertion.value}" but got "${actualType}"`,
+                };
+            }
+            case 'not_error':
+                return {
+                    assertion,
+                    passed: !isError,
+                    actualValue: isError,
+                    error: isError
+                        ? assertion.message ?? 'Expected response to not be an error'
+                        : undefined,
+                };
+            default:
+                return {
+                    assertion,
+                    passed: false,
+                    actualValue,
+                    error: `Unknown assertion condition: ${assertion.condition}`,
+                };
+        }
+    }
+    catch (error) {
+        return {
+            assertion,
+            passed: false,
+            actualValue,
+            error: `Assertion evaluation error: ${error.message}`,
+        };
+    }
+}
+/**
+ * Evaluate all assertions for a response.
+ * Extracts JSON content from MCP results for direct path access.
+ */
+export function evaluateAssertions(assertions, response, isError) {
+    if (isError && (response === null || response === undefined)) {
+        const errorResponse = {
+            error: true,
+            isError: true,
+            message: 'MCP protocol error occurred',
+        };
+        return assertions.map((assertion) => evaluateAssertion(assertion, errorResponse, isError));
+    }
+    // Extract actual content from MCP response wrapper
+    const extractedResponse = extractResponseContent(response);
+    return assertions.map((assertion) => evaluateAssertion(assertion, extractedResponse, isError));
+}
+/**
+ * Format assertion results for display.
+ */
+export function formatAssertionResults(results) {
+    const lines = [];
+    for (const result of results) {
+        const status = result.passed ? '✓' : '✗';
+        const condition = result.assertion.condition;
+        const path = result.assertion.path;
+        if (result.passed) {
+            lines.push(`  ${status} ${path} ${condition}`);
+        }
+        else {
+            lines.push(`  ${status} ${path} ${condition}: ${result.error}`);
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=evaluator.js.map

package/dist/scenarios/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Custom test scenarios module.
+ *
+ * Provides YAML-based test scenario definitions that can be used
+ * alongside LLM-generated test cases.
+ */
+export type { TestScenario, PromptScenario, ScenarioAssertion, AssertionCondition, TestScenariosYAML, LoadedScenarios, AssertionResult, ScenarioResult, } from './types.js';
+export { loadScenariosFromFile, tryLoadDefaultScenarios, generateSampleScenariosYaml, DEFAULT_SCENARIOS_FILE, } from './loader.js';
+export { evaluateAssertion, evaluateAssertions, extractResponseContent, getValueAtPath, formatAssertionResults, } from './evaluator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/scenarios/index.js

@@ -0,0 +1,9 @@
+/**
+ * Custom test scenarios module.
+ *
+ * Provides YAML-based test scenario definitions that can be used
+ * alongside LLM-generated test cases.
+ */
+export { loadScenariosFromFile, tryLoadDefaultScenarios, generateSampleScenariosYaml, DEFAULT_SCENARIOS_FILE, } from './loader.js';
+export { evaluateAssertion, evaluateAssertions, extractResponseContent, getValueAtPath, formatAssertionResults, } from './evaluator.js';
+//# sourceMappingURL=index.js.map

package/dist/scenarios/loader.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Scenario loader - loads test scenarios from YAML files.
+ */
+import type { LoadedScenarios } from './types.js';
+/** Default file name for test scenarios */
+export declare const DEFAULT_SCENARIOS_FILE: "bellwether-tests.yaml";
+/**
+ * Load test scenarios from a YAML file.
+ */
+export declare function loadScenariosFromFile(path: string): LoadedScenarios;
+/**
+ * Try to load scenarios from the default file in a directory.
+ * Returns null if file doesn't exist.
+ */
+export declare function tryLoadDefaultScenarios(directory: string): LoadedScenarios | null;
+/**
+ * Generate a sample YAML template for test scenarios.
+ */
+export declare function generateSampleScenariosYaml(): string;
+//# sourceMappingURL=loader.d.ts.map