@dotsetlabs/bellwether 0.10.0

package/dist/validation/semantic-types.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Semantic type inference and validation.
+ *
+ * This module defines semantic types that can be inferred from parameter
+ * names and descriptions, enabling more intelligent validation testing.
+ */
+/**
+ * Semantic types that can be inferred from parameter descriptions.
+ * Used to generate targeted validation tests.
+ */
+export type SemanticType = 'date_iso8601' | 'date_month' | 'datetime' | 'timestamp' | 'amount_currency' | 'percentage' | 'identifier' | 'email' | 'url' | 'phone' | 'ip_address' | 'file_path' | 'json' | 'base64' | 'regex' | 'unknown';
+/**
+ * Inference result for a parameter.
+ * Contains the inferred semantic type and confidence level.
+ */
+export interface SemanticInference {
+    /** Parameter name that was analyzed */
+    paramName: string;
+    /** The inferred semantic type */
+    inferredType: SemanticType;
+    /** Confidence level (0-1) */
+    confidence: number;
+    /** Evidence supporting the inference */
+    evidence: string[];
+}
+/**
+ * Result of validating a value against an expected semantic type.
+ */
+export interface SemanticValidationResult {
+    /** Parameter name that was validated */
+    paramName: string;
+    /** The expected semantic type */
+    expectedType: SemanticType;
+    /** The value that was validated */
+    providedValue: unknown;
+    /** Whether the value is valid for the expected type */
+    isValid: boolean;
+    /** Description of the validation issue (if invalid) */
+    issue?: string;
+}
+/**
+ * Pattern definition for semantic type inference.
+ * Contains patterns to match against parameter names and descriptions.
+ */
+export interface SemanticPatternDefinition {
+    /** Patterns to match against parameter names */
+    namePatterns: RegExp[];
+    /** Patterns to match against parameter descriptions */
+    descriptionPatterns: RegExp[];
+    /** Optional patterns to validate actual values */
+    formatPatterns?: RegExp[];
+}
+/**
+ * Patterns for semantic type inference.
+ * Maps each semantic type to patterns that indicate that type.
+ */
+export declare const SEMANTIC_PATTERNS: Record<SemanticType, SemanticPatternDefinition>;
+/**
+ * Get all known semantic types (excluding 'unknown').
+ */
+export declare function getAllSemanticTypes(): SemanticType[];
+/**
+ * Check if a value is a valid SemanticType.
+ */
+export declare function isSemanticType(value: string): value is SemanticType;
+//# sourceMappingURL=semantic-types.d.ts.map

package/dist/validation/semantic-types.js

@@ -0,0 +1,94 @@
+/**
+ * Semantic type inference and validation.
+ *
+ * This module defines semantic types that can be inferred from parameter
+ * names and descriptions, enabling more intelligent validation testing.
+ */
+/**
+ * Patterns for semantic type inference.
+ * Maps each semantic type to patterns that indicate that type.
+ */
+export const SEMANTIC_PATTERNS = {
+    date_iso8601: {
+        namePatterns: [/date/i, /day/i, /_at$/i, /created/i, /updated/i, /birth/i, /expir/i],
+        descriptionPatterns: [/YYYY-MM-DD/i, /ISO.?8601/i, /date format/i, /date string/i],
+        formatPatterns: [/^\d{4}-\d{2}-\d{2}$/],
+    },
+    date_month: {
+        namePatterns: [/month/i, /period/i, /billing_?month/i],
+        descriptionPatterns: [/YYYY-MM/i, /month format/i, /year-month/i],
+        formatPatterns: [/^\d{4}-\d{2}$/],
+    },
+    datetime: {
+        namePatterns: [/datetime/i, /timestamp/i, /time$/i, /created_?at/i, /updated_?at/i],
+        descriptionPatterns: [/ISO.?8601/i, /datetime/i, /\d{4}-\d{2}-\d{2}T/, /RFC.?3339/i],
+        formatPatterns: [/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/],
+    },
+    timestamp: {
+        namePatterns: [/timestamp/i, /unix/i, /epoch/i, /time_?ms/i, /millis/i],
+        descriptionPatterns: [/unix/i, /epoch/i, /milliseconds/i, /seconds since/i, /timestamp/i],
+    },
+    amount_currency: {
+        namePatterns: [/amount/i, /price/i, /cost/i, /fee/i, /total/i, /balance/i, /payment/i, /charge/i],
+        descriptionPatterns: [/currency/i, /dollar/i, /USD/i, /EUR/i, /amount/i, /monetary/i, /cents/i],
+    },
+    percentage: {
+        namePatterns: [/percent/i, /rate/i, /ratio/i, /progress/i, /completion/i],
+        descriptionPatterns: [/percent/i, /0-100/i, /0-1/i, /proportion/i, /percentage/i],
+    },
+    identifier: {
+        namePatterns: [/id$/i, /_id/i, /uuid/i, /guid/i, /sku/i, /code$/i],
+        descriptionPatterns: [/identifier/i, /UUID/i, /unique/i, /GUID/i],
+    },
+    email: {
+        namePatterns: [/email/i, /mail/i, /e_?mail/i],
+        descriptionPatterns: [/email/i, /@/, /e-mail/i],
+        formatPatterns: [/^[\w.-]+@[\w.-]+\.\w+$/],
+    },
+    url: {
+        namePatterns: [/url/i, /uri/i, /link/i, /href/i, /endpoint/i, /website/i],
+        descriptionPatterns: [/URL/i, /URI/i, /http/i, /link/i, /web address/i],
+        formatPatterns: [/^https?:\/\//i],
+    },
+    phone: {
+        namePatterns: [/phone/i, /tel/i, /mobile/i, /cell/i, /fax/i],
+        descriptionPatterns: [/phone/i, /telephone/i, /mobile/i, /number/i],
+    },
+    ip_address: {
+        namePatterns: [/ip/i, /ip_?address/i, /host/i, /server_?ip/i],
+        descriptionPatterns: [/IP address/i, /IPv4/i, /IPv6/i, /internet protocol/i],
+    },
+    file_path: {
+        namePatterns: [/path/i, /file/i, /directory/i, /dir/i, /folder/i, /filename/i],
+        descriptionPatterns: [/path/i, /file/i, /directory/i, /folder/i],
+    },
+    json: {
+        namePatterns: [/json/i, /data/i, /payload/i, /body/i, /config/i],
+        descriptionPatterns: [/JSON/i, /object/i, /serialized/i],
+    },
+    base64: {
+        namePatterns: [/base64/i, /encoded/i, /b64/i],
+        descriptionPatterns: [/base64/i, /encoded/i, /binary/i],
+    },
+    regex: {
+        namePatterns: [/regex/i, /pattern/i, /expression/i, /regexp/i],
+        descriptionPatterns: [/regex/i, /regular expression/i, /pattern/i, /regexp/i],
+    },
+    unknown: {
+        namePatterns: [],
+        descriptionPatterns: [],
+    },
+};
+/**
+ * Get all known semantic types (excluding 'unknown').
+ */
+export function getAllSemanticTypes() {
+    return Object.keys(SEMANTIC_PATTERNS).filter((t) => t !== 'unknown');
+}
+/**
+ * Check if a value is a valid SemanticType.
+ */
+export function isSemanticType(value) {
+    return value in SEMANTIC_PATTERNS;
+}
+//# sourceMappingURL=semantic-types.js.map

package/dist/validation/semantic-validator.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Semantic validator for inferring and validating semantic types.
+ *
+ * This module provides functions to:
+ * 1. Infer semantic types from parameter names and descriptions
+ * 2. Validate values against expected semantic types
+ */
+import type { SemanticType, SemanticInference, SemanticValidationResult } from './semantic-types.js';
+/**
+ * Infer semantic type from parameter name and description.
+ *
+ * Uses pattern matching against known semantic type patterns to determine
+ * the most likely semantic type for a parameter.
+ *
+ * @param paramName - The name of the parameter
+ * @param description - Optional description of the parameter
+ * @param schemaFormat - Optional JSON Schema format hint
+ * @returns Inference result with type, confidence, and evidence
+ */
+export declare function inferSemanticType(paramName: string, description?: string, schemaFormat?: string): SemanticInference;
+/**
+ * Validate a value against an expected semantic type.
+ *
+ * @param paramName - The name of the parameter being validated
+ * @param value - The value to validate
+ * @param expectedType - The expected semantic type
+ * @returns Validation result with validity and any issues
+ */
+export declare function validateSemanticValue(paramName: string, value: unknown, expectedType: SemanticType): SemanticValidationResult;
+/**
+ * Validate all parameters in a tool call against inferred types.
+ *
+ * @param args - The arguments to validate
+ * @param inferences - The semantic inferences for each parameter
+ * @returns Array of validation results
+ */
+export declare function validateAllParameters(args: Record<string, unknown>, inferences: SemanticInference[]): SemanticValidationResult[];
+//# sourceMappingURL=semantic-validator.d.ts.map

package/dist/validation/semantic-validator.js

@@ -0,0 +1,340 @@
+/**
+ * Semantic validator for inferring and validating semantic types.
+ *
+ * This module provides functions to:
+ * 1. Infer semantic types from parameter names and descriptions
+ * 2. Validate values against expected semantic types
+ */
+import { SEMANTIC_PATTERNS } from './semantic-types.js';
+import { SEMANTIC_VALIDATION } from '../constants.js';
+/**
+ * Infer semantic type from parameter name and description.
+ *
+ * Uses pattern matching against known semantic type patterns to determine
+ * the most likely semantic type for a parameter.
+ *
+ * @param paramName - The name of the parameter
+ * @param description - Optional description of the parameter
+ * @param schemaFormat - Optional JSON Schema format hint
+ * @returns Inference result with type, confidence, and evidence
+ */
+export function inferSemanticType(paramName, description, schemaFormat) {
+    const evidence = [];
+    let bestMatch = 'unknown';
+    let bestConfidence = 0;
+    // Check schema format first (highest confidence)
+    if (schemaFormat) {
+        const formatType = mapFormatToSemanticType(schemaFormat);
+        if (formatType !== 'unknown') {
+            return {
+                paramName,
+                inferredType: formatType,
+                confidence: SEMANTIC_VALIDATION.CONFIDENCE.SCHEMA_FORMAT,
+                evidence: [`Schema format: ${schemaFormat}`],
+            };
+        }
+    }
+    // Check each semantic type
+    for (const [type, patterns] of Object.entries(SEMANTIC_PATTERNS)) {
+        if (type === 'unknown')
+            continue;
+        let confidence = 0;
+        const typeEvidence = [];
+        // Check name patterns
+        for (const pattern of patterns.namePatterns) {
+            if (pattern.test(paramName)) {
+                confidence += SEMANTIC_VALIDATION.CONFIDENCE.NAME_PATTERN_MATCH;
+                typeEvidence.push(`Name matches pattern: ${pattern.source}`);
+                break;
+            }
+        }
+        // Check description patterns
+        if (description) {
+            for (const pattern of patterns.descriptionPatterns) {
+                if (pattern.test(description)) {
+                    confidence += SEMANTIC_VALIDATION.CONFIDENCE.DESCRIPTION_PATTERN_MATCH;
+                    typeEvidence.push(`Description matches: ${pattern.source}`);
+                    break;
+                }
+            }
+        }
+        if (confidence > bestConfidence) {
+            bestConfidence = confidence;
+            bestMatch = type;
+            evidence.length = 0;
+            evidence.push(...typeEvidence);
+        }
+    }
+    return {
+        paramName,
+        inferredType: bestMatch,
+        confidence: Math.min(1, bestConfidence),
+        evidence,
+    };
+}
+/**
+ * Map JSON Schema format to semantic type.
+ *
+ * @param format - JSON Schema format string
+ * @returns Corresponding semantic type or 'unknown'
+ */
+function mapFormatToSemanticType(format) {
+    const formatLower = format.toLowerCase();
+    const formatMap = {
+        'date': 'date_iso8601',
+        'date-time': 'datetime',
+        'email': 'email',
+        'uri': 'url',
+        'url': 'url',
+        'uuid': 'identifier',
+        'ipv4': 'ip_address',
+        'ipv6': 'ip_address',
+        'ip-address': 'ip_address',
+        'hostname': 'url',
+    };
+    return formatMap[formatLower] ?? 'unknown';
+}
+/**
+ * Validate a value against an expected semantic type.
+ *
+ * @param paramName - The name of the parameter being validated
+ * @param value - The value to validate
+ * @param expectedType - The expected semantic type
+ * @returns Validation result with validity and any issues
+ */
+export function validateSemanticValue(paramName, value, expectedType) {
+    if (expectedType === 'unknown') {
+        return { paramName, expectedType, providedValue: value, isValid: true };
+    }
+    if (typeof value !== 'string') {
+        // Non-string values can't be validated semantically
+        return { paramName, expectedType, providedValue: value, isValid: true };
+    }
+    const validators = {
+        date_iso8601: validateDate,
+        date_month: validateMonth,
+        datetime: validateDateTime,
+        timestamp: validateTimestamp,
+        amount_currency: validateAmount,
+        percentage: validatePercentage,
+        identifier: validateIdentifier,
+        email: validateEmail,
+        url: validateUrl,
+        phone: validatePhone,
+        ip_address: validateIpAddress,
+        file_path: () => null, // Paths are hard to validate generically
+        json: validateJson,
+        base64: validateBase64,
+        regex: validateRegex,
+        unknown: () => null,
+    };
+    const issue = validators[expectedType](value);
+    return {
+        paramName,
+        expectedType,
+        providedValue: value,
+        isValid: issue === null,
+        issue: issue ?? undefined,
+    };
+}
+/**
+ * Validate all parameters in a tool call against inferred types.
+ *
+ * @param args - The arguments to validate
+ * @param inferences - The semantic inferences for each parameter
+ * @returns Array of validation results
+ */
+export function validateAllParameters(args, inferences) {
+    const results = [];
+    for (const inference of inferences) {
+        const value = args[inference.paramName];
+        if (value !== undefined) {
+            results.push(validateSemanticValue(inference.paramName, value, inference.inferredType));
+        }
+    }
+    return results;
+}
+// ==================== Validation Functions ====================
+/**
+ * Validate ISO 8601 date format (YYYY-MM-DD).
+ */
+function validateDate(value) {
+    if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) {
+        return `Expected date format YYYY-MM-DD, got: ${value}`;
+    }
+    const date = new Date(value);
+    if (isNaN(date.getTime())) {
+        return `Invalid date: ${value}`;
+    }
+    // Validate the date components are valid
+    const [, month, day] = value.split('-').map(Number);
+    if (month < 1 || month > 12) {
+        return `Invalid month: ${month}`;
+    }
+    if (day < 1 || day > 31) {
+        return `Invalid day: ${day}`;
+    }
+    return null;
+}
+/**
+ * Validate month format (YYYY-MM).
+ */
+function validateMonth(value) {
+    if (!/^\d{4}-\d{2}$/.test(value)) {
+        return `Expected month format YYYY-MM, got: ${value}`;
+    }
+    const [, month] = value.split('-').map(Number);
+    if (month < 1 || month > 12) {
+        return `Invalid month: ${month}`;
+    }
+    return null;
+}
+/**
+ * Validate ISO 8601 datetime.
+ */
+function validateDateTime(value) {
+    const date = new Date(value);
+    if (isNaN(date.getTime())) {
+        return `Invalid datetime: ${value}`;
+    }
+    return null;
+}
+/**
+ * Validate Unix timestamp.
+ */
+function validateTimestamp(value) {
+    const num = Number(value);
+    if (isNaN(num) || num < 0) {
+        return `Expected positive number for timestamp, got: ${value}`;
+    }
+    return null;
+}
+/**
+ * Validate monetary amount.
+ */
+function validateAmount(value) {
+    // Remove currency symbols and commas
+    const cleaned = value.replace(/[$,\u20AC\u00A3]/g, '');
+    const num = Number(cleaned);
+    if (isNaN(num)) {
+        return `Expected numeric amount, got: ${value}`;
+    }
+    return null;
+}
+/**
+ * Validate percentage.
+ */
+function validatePercentage(value) {
+    // Remove % symbol if present
+    const cleaned = value.replace(/%$/, '');
+    const num = Number(cleaned);
+    if (isNaN(num)) {
+        return `Expected numeric percentage, got: ${value}`;
+    }
+    // Don't enforce range - could be 0-1 or 0-100
+    return null;
+}
+/**
+ * Validate identifier (non-empty string).
+ */
+function validateIdentifier(value) {
+    if (value.trim().length === 0) {
+        return 'Identifier cannot be empty';
+    }
+    return null;
+}
+/**
+ * Validate email format.
+ */
+function validateEmail(value) {
+    // Basic email validation - requires @ and at least one dot after @
+    if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value)) {
+        return `Invalid email format: ${value}`;
+    }
+    return null;
+}
+/**
+ * Validate URL format.
+ */
+function validateUrl(value) {
+    try {
+        new URL(value);
+        return null;
+    }
+    catch {
+        return `Invalid URL: ${value}`;
+    }
+}
+/**
+ * Validate phone number (loose validation - at least 7 digits).
+ */
+function validatePhone(value) {
+    const digitsOnly = value.replace(/\D/g, '');
+    if (digitsOnly.length < 7) {
+        return `Invalid phone number (too few digits): ${value}`;
+    }
+    return null;
+}
+/**
+ * Validate IP address (IPv4 or IPv6).
+ */
+function validateIpAddress(value) {
+    // IPv4 validation
+    if (/^(\d{1,3}\.){3}\d{1,3}$/.test(value)) {
+        const parts = value.split('.').map(Number);
+        if (parts.every((p) => p >= 0 && p <= 255)) {
+            return null;
+        }
+        return `Invalid IPv4 address (octets must be 0-255): ${value}`;
+    }
+    // IPv6 validation (simplified - just check for valid hex groups with colons)
+    if (/^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$/.test(value)) {
+        return null;
+    }
+    // IPv6 with :: shorthand
+    if (value.includes('::') && /^[0-9a-fA-F:]+$/.test(value)) {
+        return null;
+    }
+    return `Invalid IP address: ${value}`;
+}
+/**
+ * Validate JSON string.
+ */
+function validateJson(value) {
+    try {
+        JSON.parse(value);
+        return null;
+    }
+    catch {
+        const preview = value.length > 50 ? `${value.slice(0, 50)}...` : value;
+        return `Invalid JSON: ${preview}`;
+    }
+}
+/**
+ * Validate Base64 encoded string.
+ */
+function validateBase64(value) {
+    // Standard base64 characters plus optional padding
+    if (!/^[A-Za-z0-9+/]*={0,2}$/.test(value)) {
+        const preview = value.length > 50 ? `${value.slice(0, 50)}...` : value;
+        return `Invalid base64: ${preview}`;
+    }
+    // Check length is valid (must be multiple of 4 when padded)
+    if (value.length % 4 !== 0) {
+        return `Invalid base64 length (must be multiple of 4): ${value.length}`;
+    }
+    return null;
+}
+/**
+ * Validate regular expression.
+ */
+function validateRegex(value) {
+    try {
+        new RegExp(value);
+        return null;
+    }
+    catch {
+        return `Invalid regex: ${value}`;
+    }
+}
+//# sourceMappingURL=semantic-validator.js.map

package/dist/verification/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Verified by Bellwether program module.
+ */
+export { generateVerificationResult, generateVerificationReport, generateVerificationBadge, generateBadgeUrl, generateBadgeMarkdown, isVerificationValid, } from './verifier.js';
+export type { VerificationStatus, VerificationTier, VerificationResult, VerificationBadge, VerificationConfig, VerificationReport, VerificationSubmission, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/verification/index.js

@@ -0,0 +1,5 @@
+/**
+ * Verified by Bellwether program module.
+ */
+export { generateVerificationResult, generateVerificationReport, generateVerificationBadge, generateBadgeUrl, generateBadgeMarkdown, isVerificationValid, } from './verifier.js';
+//# sourceMappingURL=index.js.map

package/dist/verification/types.d.ts

@@ -0,0 +1,133 @@
+/**
+ * Types for the Verified by Bellwether program.
+ */
+/**
+ * Verification status for a server.
+ */
+export type VerificationStatus = 'verified' | 'pending' | 'failed' | 'expired' | 'not_verified';
+/**
+ * Verification tier based on test coverage.
+ */
+export type VerificationTier = 'bronze' | 'silver' | 'gold' | 'platinum';
+/**
+ * Verification result from a verification run.
+ */
+export interface VerificationResult {
+    /** Server identifier (namespace/name) */
+    serverId: string;
+    /** Server version */
+    version: string;
+    /** Verification status */
+    status: VerificationStatus;
+    /** Verification tier achieved */
+    tier?: VerificationTier;
+    /** ISO timestamp when verified */
+    verifiedAt: string;
+    /** ISO timestamp when verification expires */
+    expiresAt: string;
+    /** Number of tools verified */
+    toolsVerified: number;
+    /** Number of tests passed */
+    testsPassed: number;
+    /** Total number of tests run */
+    testsTotal: number;
+    /** Pass rate (0-100) */
+    passRate: number;
+    /** Checksum of the verification report */
+    reportHash: string;
+    /** Bellwether version used */
+    bellwetherVersion: string;
+    /** Additional metadata */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Verification badge data for embedding.
+ */
+export interface VerificationBadge {
+    /** Badge label */
+    label: string;
+    /** Badge message */
+    message: string;
+    /** Badge color */
+    color: string;
+    /** Verification tier icon */
+    icon?: string;
+    /** Link to verification report */
+    reportUrl?: string;
+    /** ISO timestamp of verification */
+    verifiedAt?: string;
+}
+/**
+ * Configuration for verification.
+ */
+export interface VerificationConfig {
+    /** Server identifier */
+    serverId: string;
+    /** Server version */
+    version?: string;
+    /** Tier to target */
+    targetTier?: VerificationTier;
+    /** Include security testing */
+    includeSecurity?: boolean;
+    /** Custom scenarios to include */
+    scenariosPath?: string;
+    /** Output directory for reports */
+    outputDir?: string;
+}
+/**
+ * Verification report (full detailed report).
+ */
+export interface VerificationReport {
+    /** Verification result summary */
+    result: VerificationResult;
+    /** Server information */
+    serverInfo: {
+        name: string;
+        version: string;
+        description?: string;
+        repository?: string;
+    };
+    /** Tool verification details */
+    tools: Array<{
+        name: string;
+        testsRun: number;
+        testsPassed: number;
+        errors: string[];
+    }>;
+    /** Prompt verification details */
+    prompts?: Array<{
+        name: string;
+        testsRun: number;
+        testsPassed: number;
+        errors: string[];
+    }>;
+    /** Resource verification details */
+    resources?: Array<{
+        uri: string;
+        name: string;
+        testsRun: number;
+        testsPassed: number;
+        errors: string[];
+    }>;
+    /** Environment information */
+    environment: {
+        os: string;
+        nodeVersion: string;
+        bellwetherVersion: string;
+    };
+}
+/**
+ * Verification submission for the registry.
+ */
+export interface VerificationSubmission {
+    /** Verification result */
+    result: VerificationResult;
+    /** Signed verification token */
+    signature?: string;
+    /** Submitter information */
+    submitter?: {
+        email?: string;
+        organization?: string;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/verification/types.js

@@ -0,0 +1,5 @@
+/**
+ * Types for the Verified by Bellwether program.
+ */
+export {};
+//# sourceMappingURL=types.js.map