@dotsetlabs/bellwether 0.10.0

package/dist/interview/rate-limiter.d.ts

@@ -0,0 +1,15 @@
+import type { RateLimitConfig } from './types.js';
+/**
+ * Token bucket rate limiter with async acquisition.
+ */
+export declare class RateLimiter {
+    private config;
+    private tokens;
+    private lastRefill;
+    constructor(config: RateLimitConfig);
+    acquire(): Promise<void>;
+    private refill;
+}
+export declare function isRateLimitError(message: string | null): boolean;
+export declare function calculateBackoffMs(attempt: number, strategy: RateLimitConfig['backoffStrategy']): number;
+//# sourceMappingURL=rate-limiter.d.ts.map

package/dist/interview/rate-limiter.js

@@ -0,0 +1,55 @@
+import { RATE_LIMITING } from '../constants.js';
+/**
+ * Token bucket rate limiter with async acquisition.
+ */
+export class RateLimiter {
+    config;
+    tokens;
+    lastRefill;
+    constructor(config) {
+        this.config = config;
+        this.tokens = config.burstLimit;
+        this.lastRefill = Date.now();
+    }
+    async acquire() {
+        // Token bucket algorithm - loop until a token is available
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            this.refill();
+            if (this.tokens >= 1) {
+                this.tokens -= 1;
+                return;
+            }
+            const waitMs = Math.max(1, Math.ceil(((1 - this.tokens) / this.config.requestsPerSecond) * 1000));
+            await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+    }
+    refill() {
+        const now = Date.now();
+        const elapsedSeconds = (now - this.lastRefill) / 1000;
+        if (elapsedSeconds <= 0)
+            return;
+        const refillTokens = elapsedSeconds * this.config.requestsPerSecond;
+        if (refillTokens >= 1) {
+            this.tokens = Math.min(this.config.burstLimit, this.tokens + refillTokens);
+            this.lastRefill = now;
+        }
+    }
+}
+export function isRateLimitError(message) {
+    if (!message)
+        return false;
+    return RATE_LIMITING.ERROR_PATTERNS.some((pattern) => pattern.test(message));
+}
+export function calculateBackoffMs(attempt, strategy) {
+    const baseDelay = RATE_LIMITING.BASE_DELAY_MS;
+    const maxDelay = RATE_LIMITING.MAX_DELAY_MS;
+    const jitter = RATE_LIMITING.JITTER_RATIO;
+    const rawDelay = strategy === 'linear'
+        ? baseDelay * (attempt + 1)
+        : baseDelay * Math.pow(2, attempt);
+    const capped = Math.min(rawDelay, maxDelay);
+    const jitterDelta = capped * jitter * (Math.random() - 0.5) * 2;
+    return Math.max(0, Math.round(capped + jitterDelta));
+}
+//# sourceMappingURL=rate-limiter.js.map

package/dist/interview/response-validator.d.ts

@@ -0,0 +1,10 @@
+import type { MCPToolCallResult } from '../transport/types.js';
+import type { ResponseAssertionResult, ResponseSchema } from './types.js';
+export interface ResponseAssertionOptions {
+    expectedText?: string;
+}
+/**
+ * Validate a response against an inferred schema and basic semantic assertions.
+ */
+export declare function validateResponseAssertions(response: MCPToolCallResult, expectedSchema?: ResponseSchema, options?: ResponseAssertionOptions): ResponseAssertionResult[];
+//# sourceMappingURL=response-validator.d.ts.map

package/dist/interview/response-validator.js

@@ -0,0 +1,132 @@
+import { inferSchemaFromValue, computeInferredSchemaHash } from '../baseline/response-fingerprint.js';
+import { extractTextContent } from './schema-inferrer.js';
+/**
+ * Validate a response against an inferred schema and basic semantic assertions.
+ */
+export function validateResponseAssertions(response, expectedSchema, options = {}) {
+    const results = [];
+    const textContent = extractTextContent(response);
+    const jsonContent = textContent ? tryParseJson(textContent) : null;
+    const hasBinary = response.content?.some((c) => c.type !== 'text') ?? false;
+    results.push(assertNotEmpty(textContent, jsonContent?.value, hasBinary));
+    if (expectedSchema?.inferredType === 'json') {
+        results.push(assertIsJson(jsonContent));
+        if (expectedSchema.jsonSchema) {
+            results.push(assertMatchesSchema(jsonContent, expectedSchema.jsonSchema));
+            results.push(assertContainsFields(jsonContent, expectedSchema.jsonSchema));
+        }
+    }
+    if (expectedSchema?.inferredType === 'markdown' && expectedSchema.markdownStructure) {
+        results.push(assertMarkdownStructure(textContent, expectedSchema.markdownStructure));
+    }
+    if (options.expectedText) {
+        results.push(assertContainsText(textContent, options.expectedText));
+    }
+    return results.filter(Boolean);
+}
+function assertNotEmpty(textContent, jsonValue, hasBinary) {
+    const isEmptyText = !textContent || textContent.trim().length === 0;
+    const isEmptyJson = jsonValue === undefined || jsonValue === null ||
+        (Array.isArray(jsonValue) && jsonValue.length === 0) ||
+        (typeof jsonValue === 'object' && !Array.isArray(jsonValue) && Object.keys(jsonValue).length === 0);
+    const passed = hasBinary || !(isEmptyText && isEmptyJson);
+    return {
+        type: 'not_empty',
+        passed,
+        message: passed ? undefined : 'Response was empty',
+    };
+}
+function assertIsJson(parsed) {
+    const passed = !!parsed?.success;
+    return {
+        type: 'is_json',
+        passed,
+        message: passed ? undefined : 'Response is not valid JSON',
+    };
+}
+function assertMatchesSchema(parsed, expectedSchema) {
+    if (!parsed?.success) {
+        return {
+            type: 'matches_schema',
+            passed: false,
+            message: 'Response is not valid JSON',
+        };
+    }
+    const actualSchema = inferSchemaFromValue(parsed.value);
+    const expectedHash = computeInferredSchemaHash(expectedSchema);
+    const actualHash = computeInferredSchemaHash(actualSchema);
+    const passed = expectedHash === actualHash;
+    return {
+        type: 'matches_schema',
+        passed,
+        message: passed ? undefined : 'Response schema does not match expected structure',
+        expected: expectedHash,
+        actual: actualHash,
+    };
+}
+function assertContainsFields(parsed, expectedSchema) {
+    if (!parsed?.success || parsed.value === null || typeof parsed.value !== 'object' || Array.isArray(parsed.value)) {
+        return {
+            type: 'contains_fields',
+            passed: false,
+            message: 'Response JSON is not an object',
+        };
+    }
+    const expectedProps = expectedSchema.properties ?? {};
+    const requiredFields = Object.keys(expectedProps);
+    if (requiredFields.length === 0) {
+        return {
+            type: 'contains_fields',
+            passed: true,
+        };
+    }
+    const actualFields = Object.keys(parsed.value);
+    const missing = requiredFields.filter((field) => !actualFields.includes(field));
+    return {
+        type: 'contains_fields',
+        passed: missing.length === 0,
+        message: missing.length === 0 ? undefined : `Missing fields: ${missing.join(', ')}`,
+        expected: requiredFields,
+        actual: actualFields,
+    };
+}
+function assertMarkdownStructure(textContent, expected) {
+    if (!textContent) {
+        return {
+            type: 'contains_text',
+            passed: false,
+            message: 'Response text is empty',
+        };
+    }
+    const hasHeaders = /^#{1,6}\s+/m.test(textContent);
+    const hasTables = /^\|.+\|\s*$/m.test(textContent);
+    const hasCodeBlocks = /```[\s\S]*?```/m.test(textContent);
+    const passed = (!expected.hasHeaders || hasHeaders) &&
+        (!expected.hasTables || hasTables) &&
+        (!expected.hasCodeBlocks || hasCodeBlocks);
+    return {
+        type: 'contains_text',
+        passed,
+        message: passed ? undefined : 'Markdown structure is missing expected sections',
+        expected,
+        actual: { hasHeaders, hasTables, hasCodeBlocks },
+    };
+}
+function assertContainsText(textContent, expected) {
+    const passed = !!textContent && textContent.includes(expected);
+    return {
+        type: 'contains_text',
+        passed,
+        message: passed ? undefined : `Response missing expected text: "${expected}"`,
+        expected,
+    };
+}
+function tryParseJson(text) {
+    try {
+        return { success: true, value: JSON.parse(text) };
+    }
+    catch {
+        return { success: false };
+    }
+}
+//# sourceMappingURL=response-validator.js.map

package/dist/interview/schema-inferrer.d.ts

@@ -0,0 +1,8 @@
+import type { MCPToolCallResult } from '../transport/types.js';
+import type { ResponseSchema } from './types.js';
+/**
+ * Infer a response schema from an MCP tool response.
+ */
+export declare function inferResponseSchema(response: MCPToolCallResult): ResponseSchema | null;
+export declare function extractTextContent(response: MCPToolCallResult): string | null;
+//# sourceMappingURL=schema-inferrer.d.ts.map

package/dist/interview/schema-inferrer.js

@@ -0,0 +1,71 @@
+import { createHash } from 'crypto';
+import { inferSchemaFromValue, computeInferredSchemaHash } from '../baseline/response-fingerprint.js';
+/**
+ * Infer a response schema from an MCP tool response.
+ */
+export function inferResponseSchema(response) {
+    const textContent = extractTextContent(response);
+    if (!textContent) {
+        const hasBinary = response.content?.some((c) => c.type !== 'text');
+        if (hasBinary) {
+            return {
+                inferredType: 'binary',
+                sampleFingerprints: [],
+            };
+        }
+        return null;
+    }
+    const parsedJson = tryParseJson(textContent);
+    if (parsedJson.success) {
+        const jsonSchema = inferSchemaFromValue(parsedJson.value);
+        const hash = computeInferredSchemaHash(jsonSchema);
+        return {
+            inferredType: 'json',
+            jsonSchema,
+            sampleFingerprints: [hash],
+        };
+    }
+    const markdownStructure = detectMarkdownStructure(textContent);
+    if (markdownStructure.hasHeaders || markdownStructure.hasTables || markdownStructure.hasCodeBlocks) {
+        return {
+            inferredType: 'markdown',
+            markdownStructure,
+            sampleFingerprints: [hashString(textContent)],
+        };
+    }
+    return {
+        inferredType: 'text',
+        sampleFingerprints: [hashString(textContent)],
+    };
+}
+export function extractTextContent(response) {
+    if (!response.content || response.content.length === 0) {
+        return null;
+    }
+    const textBlocks = response.content
+        .filter((c) => c.type === 'text' && typeof c.text === 'string')
+        .map((c) => c.text);
+    if (textBlocks.length === 0) {
+        return null;
+    }
+    return textBlocks.join('\n');
+}
+function tryParseJson(text) {
+    try {
+        return { success: true, value: JSON.parse(text) };
+    }
+    catch {
+        return { success: false };
+    }
+}
+function detectMarkdownStructure(text) {
+    return {
+        hasHeaders: /^#{1,6}\s+/m.test(text),
+        hasTables: /^\|.+\|\s*$/m.test(text),
+        hasCodeBlocks: /```[\s\S]*?```/m.test(text),
+    };
+}
+function hashString(value) {
+    return createHash('sha256').update(value).digest('hex');
+}
+//# sourceMappingURL=schema-inferrer.js.map

package/dist/interview/schema-test-generator.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Schema-based test generator for deterministic testing in check mode.
+ *
+ * Generates comprehensive test cases from JSON Schema without requiring LLM.
+ * This module is the core of the enhanced testing capability, producing
+ * 8-12 tests per tool covering boundaries, types, enums, and error handling.
+ */
+import type { MCPTool } from '../transport/types.js';
+import type { InterviewQuestion, ExpectedOutcome } from './types.js';
+import type { QuestionCategory } from '../persona/types.js';
+import type { SemanticInference } from '../validation/semantic-types.js';
+/**
+ * Options for test generation.
+ */
+export interface SchemaTestGeneratorOptions {
+    /** Skip error handling tests (e.g., missing required params) */
+    skipErrorTests?: boolean;
+    /** Maximum tests per tool (overrides default) */
+    maxTestsPerTool?: number;
+    /** Skip semantic validation tests (default: false) */
+    skipSemanticTests?: boolean;
+}
+/**
+ * Result of schema test generation including semantic inferences.
+ */
+export interface SchemaTestGeneratorResult {
+    /** Generated test questions */
+    questions: InterviewQuestion[];
+    /** Semantic type inferences for parameters */
+    semanticInferences: SemanticInference[];
+}
+/**
+ * Determine expected outcome for a test based on its category and description.
+ * Uses OUTCOME_ASSESSMENT constants to classify tests.
+ *
+ * @param category - Test category
+ * @param description - Test description
+ * @returns Expected outcome: 'success', 'error', or 'either'
+ */
+export declare function determineExpectedOutcome(category: QuestionCategory, description: string): ExpectedOutcome;
+/**
+ * Generate comprehensive schema-based test cases for a tool.
+ *
+ * This function analyzes the tool's JSON Schema and generates deterministic
+ * test cases covering:
+ * - Happy path (valid inputs)
+ * - Boundary values (empty strings, zero, large numbers)
+ * - Type coercion (wrong types)
+ * - Enum validation (invalid enum values)
+ * - Array handling (empty, single, many items)
+ * - Null/undefined handling
+ * - Error handling (missing required params)
+ * - Semantic validation (date formats, emails, URLs, etc.)
+ *
+ * @param tool - The MCP tool to generate tests for
+ * @param options - Configuration options
+ * @returns Array of interview questions (test cases)
+ */
+export declare function generateSchemaTests(tool: MCPTool, options?: SchemaTestGeneratorOptions): InterviewQuestion[];
+/**
+ * Generate schema-based tests with semantic inference information.
+ *
+ * This variant returns both the test questions and semantic type inferences,
+ * useful when you need to track inferred types for documentation.
+ *
+ * @param tool - The MCP tool to generate tests for
+ * @param options - Configuration options
+ * @returns Test questions and semantic inferences
+ */
+export declare function generateSchemaTestsWithInferences(tool: MCPTool, options?: SchemaTestGeneratorOptions): SchemaTestGeneratorResult;
+//# sourceMappingURL=schema-test-generator.d.ts.map