@dotsetlabs/bellwether 0.10.0

package/dist/docs/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Documentation and reporting module.
+ *
+ * Supported output formats:
+ * - CONTRACT.md: Check command output (schema-focused)
+ * - AGENTS.md: Explore command output (behavior-focused, LLM-powered)
+ * - JSON: Machine-readable interview results
+ */
+export { generateAgentsMd, generateContractMd, generateJsonReport } from './generator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/docs/index.js

@@ -0,0 +1,11 @@
+/**
+ * Documentation and reporting module.
+ *
+ * Supported output formats:
+ * - CONTRACT.md: Check command output (schema-focused)
+ * - AGENTS.md: Explore command output (behavior-focused, LLM-powered)
+ * - JSON: Machine-readable interview results
+ */
+// Main generators
+export { generateAgentsMd, generateContractMd, generateJsonReport } from './generator.js';
+//# sourceMappingURL=index.js.map

package/dist/docs/junit-reporter.d.ts

@@ -0,0 +1,18 @@
+/**
+ * JUnit XML reporter for test runner integration.
+ */
+import type { InterviewResult } from '../interview/types.js';
+import type { BehavioralDiff } from '../baseline/types.js';
+/**
+ * Generate JUnit XML report from interview results.
+ *
+ * Each tool becomes a test suite, and each interaction becomes a test case.
+ */
+export declare function generateJunitReport(result: InterviewResult): string;
+/**
+ * Generate JUnit XML report from behavioral diff.
+ *
+ * Treats drift detection as a test suite.
+ */
+export declare function generateJunitFromDiff(diff: BehavioralDiff): string;
+//# sourceMappingURL=junit-reporter.d.ts.map

package/dist/docs/junit-reporter.js

@@ -0,0 +1,210 @@
+/**
+ * JUnit XML reporter for test runner integration.
+ */
+/**
+ * Escape special XML characters.
+ */
+function escapeXml(str) {
+    return str
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+/**
+ * Generate JUnit XML report from interview results.
+ *
+ * Each tool becomes a test suite, and each interaction becomes a test case.
+ */
+export function generateJunitReport(result) {
+    const lines = [];
+    lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+    // Calculate totals
+    const totalTests = result.toolProfiles.reduce((sum, p) => sum + p.interactions.length, 0);
+    const failures = result.toolProfiles.reduce((sum, p) => sum + p.interactions.filter((i) => i.error).length, 0);
+    const durationSeconds = result.metadata.durationMs / 1000;
+    lines.push(`<testsuites name="${escapeXml(result.discovery.serverInfo.name)}" ` +
+        `tests="${totalTests}" failures="${failures}" errors="0" ` +
+        `time="${durationSeconds.toFixed(3)}">`);
+    // Each tool is a test suite
+    for (const profile of result.toolProfiles) {
+        lines.push(generateToolSuite(profile));
+    }
+    // Add workflow test suites if present
+    if (result.workflowResults && result.workflowResults.length > 0) {
+        lines.push(generateWorkflowSuites(result));
+    }
+    lines.push('</testsuites>');
+    return lines.join('\n');
+}
+/**
+ * Generate a test suite for a single tool.
+ */
+function generateToolSuite(profile) {
+    const lines = [];
+    const tests = profile.interactions.length;
+    const failures = profile.interactions.filter((i) => i.error).length;
+    const duration = profile.interactions.reduce((sum, i) => sum + i.durationMs, 0) / 1000;
+    lines.push(`  <testsuite name="${escapeXml(profile.name)}" ` +
+        `tests="${tests}" failures="${failures}" errors="0" ` +
+        `time="${duration.toFixed(3)}" ` +
+        `timestamp="${new Date().toISOString()}">`);
+    // Tool description as property
+    lines.push('    <properties>');
+    lines.push(`      <property name="description" value="${escapeXml(profile.description)}"/>`);
+    lines.push('    </properties>');
+    // Each interaction is a test case
+    for (const interaction of profile.interactions) {
+        lines.push(generateTestCase(profile.name, interaction));
+    }
+    // Add behavioral note test cases (pass by default)
+    for (let i = 0; i < profile.behavioralNotes.length; i++) {
+        lines.push(`    <testcase name="behavioral_note_${i + 1}" classname="${escapeXml(profile.name)}" time="0">`);
+        lines.push(`      <system-out>${escapeXml(profile.behavioralNotes[i])}</system-out>`);
+        lines.push('    </testcase>');
+    }
+    // Add security notes as warnings
+    for (let i = 0; i < profile.securityNotes.length; i++) {
+        lines.push(`    <testcase name="security_note_${i + 1}" classname="${escapeXml(profile.name)}" time="0">`);
+        lines.push(`      <system-err>Security consideration: ${escapeXml(profile.securityNotes[i])}</system-err>`);
+        lines.push('    </testcase>');
+    }
+    lines.push('  </testsuite>');
+    return lines.join('\n');
+}
+/**
+ * Generate a test case for a single interaction.
+ */
+function generateTestCase(toolName, interaction) {
+    const lines = [];
+    const duration = interaction.durationMs / 1000;
+    const testName = escapeXml(interaction.question.description);
+    const className = escapeXml(toolName);
+    lines.push(`    <testcase name="${testName}" classname="${className}" time="${duration.toFixed(3)}">`);
+    if (interaction.error) {
+        lines.push('      <failure message="Tool call failed">');
+        lines.push(`        ${escapeXml(interaction.error)}`);
+        lines.push('      </failure>');
+    }
+    // Include the analysis as system-out
+    if (interaction.analysis) {
+        lines.push(`      <system-out>${escapeXml(interaction.analysis)}</system-out>`);
+    }
+    lines.push('    </testcase>');
+    return lines.join('\n');
+}
+/**
+ * Generate test suites for workflows.
+ */
+function generateWorkflowSuites(result) {
+    const lines = [];
+    if (!result.workflowResults)
+        return '';
+    for (const wr of result.workflowResults) {
+        const tests = wr.steps.length;
+        const failures = wr.steps.filter((s) => !s.success).length;
+        lines.push(`  <testsuite name="workflow:${escapeXml(wr.workflow.name)}" ` +
+            `tests="${tests}" failures="${failures}" errors="0" time="0">`);
+        lines.push('    <properties>');
+        lines.push(`      <property name="description" value="${escapeXml(wr.workflow.description)}"/>`);
+        lines.push(`      <property name="expectedOutcome" value="${escapeXml(wr.workflow.expectedOutcome)}"/>`);
+        lines.push('    </properties>');
+        // Each step is a test case
+        for (let i = 0; i < wr.steps.length; i++) {
+            const stepResult = wr.steps[i];
+            const step = stepResult.step;
+            lines.push(`    <testcase name="step_${i + 1}:${escapeXml(step.tool)}" ` +
+                `classname="workflow:${escapeXml(wr.workflow.name)}" time="0">`);
+            if (!stepResult.success) {
+                lines.push('      <failure message="Step failed">');
+                if (stepResult.error) {
+                    lines.push(`        ${escapeXml(stepResult.error)}`);
+                }
+                lines.push('      </failure>');
+            }
+            if (stepResult.analysis) {
+                lines.push(`      <system-out>${escapeXml(stepResult.analysis)}</system-out>`);
+            }
+            lines.push('    </testcase>');
+        }
+        lines.push('  </testsuite>');
+    }
+    return lines.join('\n');
+}
+/**
+ * Generate JUnit XML report from behavioral diff.
+ *
+ * Treats drift detection as a test suite.
+ */
+export function generateJunitFromDiff(diff) {
+    const lines = [];
+    lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+    const tests = diff.toolsRemoved.length +
+        diff.toolsAdded.length +
+        diff.toolsModified.length +
+        diff.behaviorChanges.length;
+    const failures = diff.breakingCount;
+    const errors = diff.toolsRemoved.length;
+    lines.push(`<testsuites name="Behavioral Drift Detection" ` +
+        `tests="${tests}" failures="${failures}" errors="${errors}" time="0">`);
+    // Tool changes suite
+    lines.push(`  <testsuite name="tool_changes" tests="${diff.toolsRemoved.length + diff.toolsAdded.length + diff.toolsModified.length}" failures="0" errors="${diff.toolsRemoved.length}">`);
+    // Removed tools are errors
+    for (const tool of diff.toolsRemoved) {
+        lines.push(`    <testcase name="tool_present:${escapeXml(tool)}" classname="drift.tools">`);
+        lines.push('      <error message="Tool was removed">');
+        lines.push(`        Tool "${escapeXml(tool)}" is no longer present on the server`);
+        lines.push('      </error>');
+        lines.push('    </testcase>');
+    }
+    // Added tools are pass with info
+    for (const tool of diff.toolsAdded) {
+        lines.push(`    <testcase name="new_tool:${escapeXml(tool)}" classname="drift.tools">`);
+        lines.push(`      <system-out>New tool discovered: ${escapeXml(tool)}</system-out>`);
+        lines.push('    </testcase>');
+    }
+    // Modified tools
+    for (const toolDiff of diff.toolsModified) {
+        const hasHighChanges = toolDiff.changes.some((c) => c.significance === 'high');
+        lines.push(`    <testcase name="tool_unchanged:${escapeXml(toolDiff.tool)}" classname="drift.tools">`);
+        if (hasHighChanges || toolDiff.schemaChanged) {
+            lines.push('      <failure message="Tool behavior changed">');
+            if (toolDiff.schemaChanged) {
+                lines.push('        Schema changed');
+            }
+            for (const change of toolDiff.changes) {
+                lines.push(`        ${escapeXml(change.description)}`);
+            }
+            lines.push('      </failure>');
+        }
+        else {
+            lines.push(`      <system-out>Minor changes detected</system-out>`);
+        }
+        lines.push('    </testcase>');
+    }
+    lines.push('  </testsuite>');
+    // Behavior changes suite
+    if (diff.behaviorChanges.length > 0) {
+        const behaviorFailures = diff.behaviorChanges.filter((c) => c.significance === 'high').length;
+        lines.push(`  <testsuite name="behavior_changes" tests="${diff.behaviorChanges.length}" failures="${behaviorFailures}" errors="0">`);
+        for (let i = 0; i < diff.behaviorChanges.length; i++) {
+            const change = diff.behaviorChanges[i];
+            lines.push(`    <testcase name="behavior_${i + 1}:${escapeXml(change.tool)}" classname="drift.behavior.${escapeXml(change.aspect)}">`);
+            if (change.significance === 'high') {
+                lines.push(`      <failure message="${escapeXml(change.description)}">`);
+                lines.push(`        Before: ${escapeXml(change.before || '(none)')}`);
+                lines.push(`        After: ${escapeXml(change.after || '(none)')}`);
+                lines.push('      </failure>');
+            }
+            else {
+                lines.push(`      <system-out>${escapeXml(change.description)}</system-out>`);
+            }
+            lines.push('    </testcase>');
+        }
+        lines.push('  </testsuite>');
+    }
+    lines.push('</testsuites>');
+    return lines.join('\n');
+}
+//# sourceMappingURL=junit-reporter.js.map

package/dist/docs/report.d.ts

@@ -0,0 +1,14 @@
+import type { InterviewResult } from '../interview/types.js';
+export interface JsonReportOptions {
+    /** Schema URL to embed in the report */
+    schemaUrl?: string;
+    /** Path to the schema file for validation */
+    schemaPath?: string;
+    /** Validate output against schema before writing */
+    validate?: boolean;
+}
+/**
+ * Generate a JSON report of the interview.
+ */
+export declare function generateJsonReport(result: InterviewResult, options?: JsonReportOptions): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/docs/report.js

@@ -0,0 +1,44 @@
+import { Ajv2020 as Ajv } from 'ajv/dist/2020.js';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { REPORT_SCHEMAS } from '../constants.js';
+/**
+ * Generate a JSON report of the interview.
+ */
+export function generateJsonReport(result, options = {}) {
+    const report = options.schemaUrl
+        ? { $schema: options.schemaUrl, ...result }
+        : { ...result };
+    const jsonReadyReport = JSON.parse(JSON.stringify(report));
+    if (options.validate) {
+        const schemaPath = resolveSchemaPath(options.schemaPath);
+        validateReportAgainstSchema(jsonReadyReport, schemaPath);
+    }
+    return JSON.stringify(jsonReadyReport, null, 2);
+}
+function resolveSchemaPath(schemaPath) {
+    if (schemaPath) {
+        return schemaPath;
+    }
+    const url = new URL(`../../${REPORT_SCHEMAS.CHECK_REPORT_SCHEMA_FILE}`, import.meta.url);
+    return fileURLToPath(url);
+}
+function validateReportAgainstSchema(report, schemaPath) {
+    const rawSchema = readFileSync(schemaPath, 'utf-8');
+    const schema = JSON.parse(rawSchema);
+    const ajv = new Ajv({
+        allErrors: true,
+        strict: false,
+        allowUnionTypes: true,
+        // Avoid warnings for date-time without adding extra dependencies.
+        formats: {
+            'date-time': true,
+        },
+    });
+    const validate = ajv.compile(schema);
+    if (!validate(report)) {
+        const errorText = ajv.errorsText(validate.errors, { separator: '\n' });
+        throw new Error(`Check report schema validation failed:\n${errorText}`);
+    }
+}
+//# sourceMappingURL=report.js.map

package/dist/docs/sarif-reporter.d.ts

@@ -0,0 +1,19 @@
+/**
+ * SARIF (Static Analysis Results Interchange Format) reporter.
+ * Produces output compatible with GitHub Code Scanning.
+ */
+import type { InterviewResult } from '../interview/types.js';
+import type { BehavioralDiff, CIFinding } from '../baseline/types.js';
+/**
+ * Generate SARIF report from interview results.
+ */
+export declare function generateSarifReport(result: InterviewResult, serverUri?: string): string;
+/**
+ * Generate SARIF report from behavioral diff.
+ */
+export declare function generateSarifFromDiff(diff: BehavioralDiff, serverUri?: string): string;
+/**
+ * Generate SARIF from CI findings.
+ */
+export declare function generateSarifFromFindings(findings: CIFinding[], serverUri?: string): string;
+//# sourceMappingURL=sarif-reporter.d.ts.map

package/dist/docs/sarif-reporter.js

@@ -0,0 +1,335 @@
+/**
+ * SARIF (Static Analysis Results Interchange Format) reporter.
+ * Produces output compatible with GitHub Code Scanning.
+ */
+import { URLS } from '../constants.js';
+import { VERSION } from '../version.js';
+/**
+ * SARIF schema version.
+ */
+const SARIF_VERSION = '2.1.0';
+/**
+ * Bellwether rule definitions.
+ */
+const BELLWETHER_RULES = [
+    {
+        id: 'BELLWETHER-001',
+        name: 'SecurityFinding',
+        shortDescription: { text: 'Security consideration identified' },
+        fullDescription: { text: 'A security-related behavior was observed during tool testing' },
+        defaultConfiguration: { level: 'warning' },
+        helpUri: `${URLS.DOCS_BASE}#security`,
+    },
+    {
+        id: 'BELLWETHER-002',
+        name: 'BehavioralLimitation',
+        shortDescription: { text: 'Tool limitation discovered' },
+        fullDescription: { text: 'A limitation in tool behavior was identified' },
+        defaultConfiguration: { level: 'note' },
+        helpUri: `${URLS.DOCS_BASE}#limitations`,
+    },
+    {
+        id: 'BELLWETHER-003',
+        name: 'ToolRemoved',
+        shortDescription: { text: 'Tool removed from server' },
+        fullDescription: { text: 'A tool that existed in the baseline is no longer present' },
+        defaultConfiguration: { level: 'error' },
+        helpUri: `${URLS.DOCS_BASE}#drift`,
+    },
+    {
+        id: 'BELLWETHER-004',
+        name: 'ToolAdded',
+        shortDescription: { text: 'New tool added to server' },
+        fullDescription: { text: 'A new tool was discovered that was not in the baseline' },
+        defaultConfiguration: { level: 'note' },
+        helpUri: `${URLS.DOCS_BASE}#drift`,
+    },
+    {
+        id: 'BELLWETHER-005',
+        name: 'SchemaChanged',
+        shortDescription: { text: 'Tool schema changed' },
+        fullDescription: { text: 'The input schema for a tool has changed' },
+        defaultConfiguration: { level: 'warning' },
+        helpUri: `${URLS.DOCS_BASE}#drift`,
+    },
+    {
+        id: 'BELLWETHER-006',
+        name: 'BehaviorChanged',
+        shortDescription: { text: 'Tool behavior changed' },
+        fullDescription: { text: 'Observable tool behavior differs from baseline' },
+        defaultConfiguration: { level: 'warning' },
+        helpUri: `${URLS.DOCS_BASE}#drift`,
+    },
+    {
+        id: 'BELLWETHER-007',
+        name: 'WorkflowFailed',
+        shortDescription: { text: 'Workflow execution failed' },
+        fullDescription: { text: 'A workflow that previously succeeded now fails' },
+        defaultConfiguration: { level: 'error' },
+        helpUri: `${URLS.DOCS_BASE}#workflows`,
+    },
+];
+/**
+ * Generate SARIF report from interview results.
+ */
+export function generateSarifReport(result, serverUri = 'mcp-server') {
+    const results = [];
+    // Add security findings
+    for (const profile of result.toolProfiles) {
+        for (const note of profile.securityNotes) {
+            results.push({
+                ruleId: 'BELLWETHER-001',
+                level: 'warning',
+                message: { text: `${profile.name}: ${note}` },
+                locations: [
+                    {
+                        physicalLocation: {
+                            artifactLocation: { uri: serverUri },
+                        },
+                    },
+                ],
+                properties: {
+                    tool: profile.name,
+                    category: 'security',
+                },
+            });
+        }
+    }
+    // Add limitations
+    for (const profile of result.toolProfiles) {
+        for (const limitation of profile.limitations) {
+            results.push({
+                ruleId: 'BELLWETHER-002',
+                level: 'note',
+                message: { text: `${profile.name}: ${limitation}` },
+                locations: [
+                    {
+                        physicalLocation: {
+                            artifactLocation: { uri: serverUri },
+                        },
+                    },
+                ],
+                properties: {
+                    tool: profile.name,
+                    category: 'limitation',
+                },
+            });
+        }
+    }
+    // Add workflow failures
+    if (result.workflowResults) {
+        for (const wr of result.workflowResults) {
+            if (!wr.success) {
+                results.push({
+                    ruleId: 'BELLWETHER-007',
+                    level: 'error',
+                    message: {
+                        text: `Workflow "${wr.workflow.name}" failed: ${wr.failureReason || 'Unknown error'}`,
+                    },
+                    properties: {
+                        workflow: wr.workflow.id,
+                        category: 'workflow',
+                    },
+                });
+            }
+        }
+    }
+    const report = {
+        $schema: URLS.SARIF_SCHEMA,
+        version: SARIF_VERSION,
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'Bellwether',
+                        version: VERSION,
+                        informationUri: URLS.DOCS_BASE,
+                        rules: BELLWETHER_RULES,
+                    },
+                },
+                results,
+                invocations: [
+                    {
+                        executionSuccessful: true,
+                    },
+                ],
+            },
+        ],
+    };
+    return JSON.stringify(report, null, 2);
+}
+/**
+ * Generate SARIF report from behavioral diff.
+ */
+export function generateSarifFromDiff(diff, serverUri = 'mcp-server') {
+    const results = [];
+    // Removed tools
+    for (const tool of diff.toolsRemoved) {
+        results.push({
+            ruleId: 'BELLWETHER-003',
+            level: 'error',
+            message: { text: `Tool "${tool}" was removed from the server` },
+            locations: [
+                {
+                    physicalLocation: {
+                        artifactLocation: { uri: serverUri },
+                    },
+                },
+            ],
+            properties: {
+                tool,
+                category: 'drift',
+            },
+        });
+    }
+    // Added tools
+    for (const tool of diff.toolsAdded) {
+        results.push({
+            ruleId: 'BELLWETHER-004',
+            level: 'note',
+            message: { text: `New tool "${tool}" was added to the server` },
+            locations: [
+                {
+                    physicalLocation: {
+                        artifactLocation: { uri: serverUri },
+                    },
+                },
+            ],
+            properties: {
+                tool,
+                category: 'drift',
+            },
+        });
+    }
+    // Modified tools
+    for (const toolDiff of diff.toolsModified) {
+        if (toolDiff.schemaChanged) {
+            results.push({
+                ruleId: 'BELLWETHER-005',
+                level: 'warning',
+                message: { text: `Schema changed for tool "${toolDiff.tool}"` },
+                locations: [
+                    {
+                        physicalLocation: {
+                            artifactLocation: { uri: serverUri },
+                        },
+                    },
+                ],
+                properties: {
+                    tool: toolDiff.tool,
+                    category: 'drift',
+                },
+            });
+        }
+        for (const change of toolDiff.changes) {
+            const level = change.significance === 'high' ? 'error' :
+                change.significance === 'medium' ? 'warning' : 'note';
+            results.push({
+                ruleId: 'BELLWETHER-006',
+                level,
+                message: { text: change.description },
+                locations: [
+                    {
+                        physicalLocation: {
+                            artifactLocation: { uri: serverUri },
+                        },
+                    },
+                ],
+                properties: {
+                    tool: change.tool,
+                    aspect: change.aspect,
+                    significance: change.significance,
+                    category: 'drift',
+                },
+            });
+        }
+    }
+    const report = {
+        $schema: URLS.SARIF_SCHEMA,
+        version: SARIF_VERSION,
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'Bellwether',
+                        version: VERSION,
+                        informationUri: URLS.DOCS_BASE,
+                        rules: BELLWETHER_RULES,
+                    },
+                },
+                results,
+                invocations: [
+                    {
+                        executionSuccessful: diff.severity !== 'breaking',
+                        exitCode: diff.severity === 'breaking' ? 1 : 0,
+                    },
+                ],
+            },
+        ],
+    };
+    return JSON.stringify(report, null, 2);
+}
+/**
+ * Generate SARIF from CI findings.
+ */
+export function generateSarifFromFindings(findings, serverUri = 'mcp-server') {
+    const results = findings.map((finding) => {
+        const level = finding.severity === 'critical' || finding.severity === 'high' ? 'error' :
+            finding.severity === 'medium' ? 'warning' : 'note';
+        const ruleId = findingCategoryToRuleId(finding.category);
+        return {
+            ruleId,
+            level,
+            message: { text: finding.description },
+            locations: [
+                {
+                    physicalLocation: {
+                        artifactLocation: { uri: serverUri },
+                    },
+                },
+            ],
+            properties: {
+                findingId: finding.id,
+                category: finding.category,
+                severity: finding.severity,
+                tool: finding.tool,
+                recommendation: finding.recommendation,
+            },
+        };
+    });
+    const report = {
+        $schema: URLS.SARIF_SCHEMA,
+        version: SARIF_VERSION,
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'Bellwether',
+                        version: VERSION,
+                        informationUri: URLS.DOCS_BASE,
+                        rules: BELLWETHER_RULES,
+                    },
+                },
+                results,
+            },
+        ],
+    };
+    return JSON.stringify(report, null, 2);
+}
+/**
+ * Map finding category to SARIF rule ID.
+ */
+function findingCategoryToRuleId(category) {
+    switch (category) {
+        case 'security':
+            return 'BELLWETHER-001';
+        case 'reliability':
+            return 'BELLWETHER-002';
+        case 'drift':
+            return 'BELLWETHER-006';
+        case 'behavior':
+        default:
+            return 'BELLWETHER-002';
+    }
+}
+//# sourceMappingURL=sarif-reporter.js.map