@anyway-sh/node-server-sdk 0.22.8

package/dist/src/lib/generated/evaluators/mbt-evaluators.d.ts

@@ -0,0 +1,386 @@
+import type { EvaluatorWithConfig } from '../../interfaces/experiment.interface';
+import type { components } from './types';
+import { type EvaluatorSlug, type EvaluatorSchema } from './registry';
+export type AgentFlowQualityConfig = components['schemas']['request.AgentFlowQualityRequest']['config'];
+export type AgentGoalCompletenessConfig = components['schemas']['request.AgentGoalCompletenessRequest']['config'];
+export type AgentToolTrajectoryConfig = components['schemas']['request.AgentToolTrajectoryRequest']['config'];
+export type ContextRelevanceConfig = components['schemas']['request.ContextRelevanceRequest']['config'];
+export type JsonValidatorConfig = components['schemas']['request.JSONValidatorRequest']['config'];
+export type PiiDetectorConfig = components['schemas']['request.PIIDetectorRequest']['config'];
+export type PlaceholderRegexConfig = components['schemas']['request.PlaceholderRegexRequest']['config'];
+export type PromptInjectionConfig = components['schemas']['request.PromptInjectionRequest']['config'];
+export type RegexValidatorConfig = components['schemas']['request.RegexValidatorRequest']['config'];
+export type SexismDetectorConfig = components['schemas']['request.SexismDetectorRequest']['config'];
+export type ToxicityDetectorConfig = components['schemas']['request.ToxicityDetectorRequest']['config'];
+/**
+ * Create an evaluator configuration object.
+ */
+export declare function createEvaluator(slug: EvaluatorSlug, options?: {
+    version?: string;
+    config?: Record<string, unknown>;
+}): EvaluatorWithConfig;
+/**
+ * Validate that required input fields are present in task output.
+ */
+export declare function validateEvaluatorInput(slug: EvaluatorSlug, taskOutput: Record<string, unknown>): {
+    valid: boolean;
+    missingFields: string[];
+};
+/**
+ * Get all available evaluator slugs.
+ */
+export declare function getAvailableEvaluatorSlugs(): EvaluatorSlug[];
+/**
+ * Get schema information for an evaluator.
+ */
+export declare function getEvaluatorSchemaInfo(slug: EvaluatorSlug): EvaluatorSchema | undefined;
+/**
+ * Factory class for creating type-safe MBT evaluator configurations.
+ *
+ * @example
+ * ```typescript
+ * import { EvaluatorMadeByTraceloop } from '@anyway-sh/node-server-sdk';
+ *
+ * const evaluators = [
+ *   EvaluatorMadeByTraceloop.piiDetector({ probability_threshold: 0.8 }),
+ *   EvaluatorMadeByTraceloop.faithfulness(),
+ * ];
+ * ```
+ */
+export declare class EvaluatorMadeByTraceloop {
+    static create(slug: EvaluatorSlug, options?: {
+        version?: string;
+        config?: Record<string, unknown>;
+    }): EvaluatorWithConfig;
+    static getAvailableSlugs(): EvaluatorSlug[];
+    static isValidSlug(slug: string): slug is EvaluatorSlug;
+    /**
+     * Evaluate agent efficiency - detect redundant calls, unnecessary follow-ups
+  
+  **Request Body:**
+  - `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
+  - `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
+     * Required task output fields: trajectory_completions, trajectory_prompts
+     */
+    static agentEfficiency(): EvaluatorWithConfig;
+    /**
+     * Validate agent trajectory against user-defined conditions
+  
+  **Request Body:**
+  - `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
+  - `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
+  - `config.conditions` (array of strings, required): Array of evaluation conditions/rules to validate against
+  - `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)
+     * Required task output fields: trajectory_completions, trajectory_prompts
+     */
+    static agentFlowQuality(config?: AgentFlowQualityConfig): EvaluatorWithConfig;
+    /**
+     * Evaluate agent goal accuracy
+  
+  **Request Body:**
+  - `input.question` (string, required): The original question or goal
+  - `input.completion` (string, required): The agent's completion/response
+  - `input.reference` (string, required): The expected reference answer
+     * Required task output fields: completion, question, reference
+     */
+    static agentGoalAccuracy(): EvaluatorWithConfig;
+    /**
+     * Measure if agent accomplished all user goals
+  
+  **Request Body:**
+  - `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
+  - `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
+  - `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)
+     * Required task output fields: trajectory_completions, trajectory_prompts
+     */
+    static agentGoalCompleteness(config?: AgentGoalCompletenessConfig): EvaluatorWithConfig;
+    /**
+     * Detect errors or failures during tool execution
+  
+  **Request Body:**
+  - `input.tool_input` (string, required): JSON string of the tool input
+  - `input.tool_output` (string, required): JSON string of the tool output
+     * Required task output fields: tool_input, tool_output
+     */
+    static agentToolErrorDetector(): EvaluatorWithConfig;
+    /**
+     * Compare actual tool calls against expected reference tool calls
+  
+  **Request Body:**
+  - `input.executed_tool_calls` (string, required): JSON array of actual tool calls made by the agent
+  - `input.expected_tool_calls` (string, required): JSON array of expected/reference tool calls
+  - `config.threshold` (float, optional): Score threshold for pass/fail determination (default: 0.5)
+  - `config.mismatch_sensitive` (bool, optional): Whether tool calls must match exactly (default: false)
+  - `config.order_sensitive` (bool, optional): Whether order of tool calls matters (default: false)
+  - `config.input_params_sensitive` (bool, optional): Whether to compare input parameters (default: true)
+     * Required task output fields: executed_tool_calls, expected_tool_calls
+     */
+    static agentToolTrajectory(config?: AgentToolTrajectoryConfig): EvaluatorWithConfig;
+    /**
+     * Evaluate whether the answer is complete and contains all the necessary information
+  
+  **Request Body:**
+  - `input.question` (string, required): The original question
+  - `input.completion` (string, required): The completion to evaluate for completeness
+  - `input.context` (string, required): The context that provides the complete information
+     * Required task output fields: completion, context, question
+     */
+    static answerCompleteness(): EvaluatorWithConfig;
+    /**
+     * Evaluate factual accuracy by comparing answers against ground truth
+  
+  **Request Body:**
+  - `input.question` (string, required): The original question
+  - `input.completion` (string, required): The completion to evaluate
+  - `input.ground_truth` (string, required): The expected correct answer
+     * Required task output fields: completion, ground_truth, question
+     */
+    static answerCorrectness(): EvaluatorWithConfig;
+    /**
+     * Check if an answer is relevant to a question
+  
+  **Request Body:**
+  - `input.answer` (string, required): The answer to evaluate for relevancy
+  - `input.question` (string, required): The question that the answer should be relevant to
+     * Required task output fields: answer, question
+     */
+    static answerRelevancy(): EvaluatorWithConfig;
+    /**
+     * Count the number of characters in text
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to count characters in
+     * Required task output fields: text
+     */
+    static charCount(): EvaluatorWithConfig;
+    /**
+     * Calculate the ratio of characters between two texts
+  
+  **Request Body:**
+  - `input.numerator_text` (string, required): The numerator text (will be divided by denominator)
+  - `input.denominator_text` (string, required): The denominator text (divides the numerator)
+     * Required task output fields: denominator_text, numerator_text
+     */
+    static charCountRatio(): EvaluatorWithConfig;
+    /**
+     * Evaluate whether retrieved context contains sufficient information to answer the query
+  
+  **Request Body:**
+  - `input.query` (string, required): The query/question to evaluate context relevance for
+  - `input.context` (string, required): The context to evaluate for relevance to the query
+  - `config.model` (string, optional): Model to use for evaluation (default: gpt-4o)
+     * Required task output fields: context, query
+     */
+    static contextRelevance(config?: ContextRelevanceConfig): EvaluatorWithConfig;
+    /**
+     * Evaluate conversation quality based on tone, clarity, flow, responsiveness, and transparency
+  
+  **Request Body:**
+  - `input.prompts` (string, required): JSON array of prompts in the conversation
+  - `input.completions` (string, required): JSON array of completions in the conversation
+     * Required task output fields: completions, prompts
+     */
+    static conversationQuality(): EvaluatorWithConfig;
+    /**
+     * Check if a completion is faithful to the provided context
+  
+  **Request Body:**
+  - `input.completion` (string, required): The LLM completion to check for faithfulness
+  - `input.context` (string, required): The context that the completion should be faithful to
+  - `input.question` (string, required): The original question asked
+     * Required task output fields: completion, context, question
+     */
+    static faithfulness(): EvaluatorWithConfig;
+    /**
+     * Compare two HTML documents for structural and content similarity
+  
+  **Request Body:**
+  - `input.html1` (string, required): The first HTML document to compare
+  - `input.html2` (string, required): The second HTML document to compare
+     * Required task output fields: html1, html2
+     */
+    static htmlComparison(): EvaluatorWithConfig;
+    /**
+     * Evaluate how well responses follow given instructions
+  
+  **Request Body:**
+  - `input.instructions` (string, required): The instructions that should be followed
+  - `input.response` (string, required): The response to evaluate for instruction adherence
+     * Required task output fields: instructions, response
+     */
+    static instructionAdherence(): EvaluatorWithConfig;
+    /**
+     * Detect changes in user intent between prompts and completions
+  
+  **Request Body:**
+  - `input.prompts` (string, required): JSON array of prompts in the conversation
+  - `input.completions` (string, required): JSON array of completions in the conversation
+     * Required task output fields: completions, prompts
+     */
+    static intentChange(): EvaluatorWithConfig;
+    /**
+     * Validate JSON syntax
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to validate as JSON
+  - `config.enable_schema_validation` (bool, optional): Enable JSON schema validation
+  - `config.schema_string` (string, optional): JSON schema to validate against
+     * Required task output fields: text
+     */
+    static jsonValidator(config?: JsonValidatorConfig): EvaluatorWithConfig;
+    /**
+     * Measure text perplexity from logprobs
+  
+  **Request Body:**
+  - `input.logprobs` (string, required): JSON array of log probabilities from the model
+     * Required task output fields: logprobs
+     */
+    static perplexity(): EvaluatorWithConfig;
+    /**
+     * Detect personally identifiable information in text
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to scan for personally identifiable information
+  - `config.probability_threshold` (float, optional): Detection threshold (default: 0.8)
+     * Required task output fields: text
+     */
+    static piiDetector(config?: PiiDetectorConfig): EvaluatorWithConfig;
+    /**
+     * Validate text against a placeholder regex pattern
+  
+  **Request Body:**
+  - `input.placeholder_value` (string, required): The regex pattern to match against
+  - `input.text` (string, required): The text to validate against the regex pattern
+  - `config.should_match` (bool, optional): Whether the text should match the regex
+  - `config.case_sensitive` (bool, optional): Case-sensitive matching
+  - `config.dot_include_nl` (bool, optional): Dot matches newlines
+  - `config.multi_line` (bool, optional): Multi-line mode
+     * Required task output fields: placeholder_value, text
+     */
+    static placeholderRegex(config?: PlaceholderRegexConfig): EvaluatorWithConfig;
+    /**
+     * Detect profanity in text
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to scan for profanity
+     * Required task output fields: text
+     */
+    static profanityDetector(): EvaluatorWithConfig;
+    /**
+     * Detect prompt injection attempts
+  
+  **Request Body:**
+  - `input.prompt` (string, required): The prompt to check for injection attempts
+  - `config.threshold` (float, optional): Detection threshold (default: 0.5)
+     * Required task output fields: prompt
+     */
+    static promptInjection(config?: PromptInjectionConfig): EvaluatorWithConfig;
+    /**
+     * Measure prompt perplexity to detect potential injection attempts
+  
+  **Request Body:**
+  - `input.prompt` (string, required): The prompt to calculate perplexity for
+     * Required task output fields: prompt
+     */
+    static promptPerplexity(): EvaluatorWithConfig;
+    /**
+     * Validate text against a regex pattern
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to validate against a regex pattern
+  - `config.regex` (string, optional): The regex pattern to match against
+  - `config.should_match` (bool, optional): Whether the text should match the regex
+  - `config.case_sensitive` (bool, optional): Case-sensitive matching
+  - `config.dot_include_nl` (bool, optional): Dot matches newlines
+  - `config.multi_line` (bool, optional): Multi-line mode
+     * Required task output fields: text
+     */
+    static regexValidator(config?: RegexValidatorConfig): EvaluatorWithConfig;
+    /**
+     * Detect secrets and credentials in text
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to scan for secrets (API keys, passwords, etc.)
+     * Required task output fields: text
+     */
+    static secretsDetector(): EvaluatorWithConfig;
+    /**
+     * Calculate semantic similarity between completion and reference
+  
+  **Request Body:**
+  - `input.completion` (string, required): The completion text to compare
+  - `input.reference` (string, required): The reference text to compare against
+     * Required task output fields: completion, reference
+     */
+    static semanticSimilarity(): EvaluatorWithConfig;
+    /**
+     * Detect sexist language and bias
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to scan for sexist content
+  - `config.threshold` (float, optional): Detection threshold (default: 0.5)
+     * Required task output fields: text
+     */
+    static sexismDetector(config?: SexismDetectorConfig): EvaluatorWithConfig;
+    /**
+     * Validate SQL query syntax
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to validate as SQL
+     * Required task output fields: text
+     */
+    static sqlValidator(): EvaluatorWithConfig;
+    /**
+     * Detect the tone of the text
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to detect the tone of
+     * Required task output fields: text
+     */
+    static toneDetection(): EvaluatorWithConfig;
+    /**
+     * Evaluate topic adherence
+  
+  **Request Body:**
+  - `input.question` (string, required): The original question
+  - `input.completion` (string, required): The completion to evaluate
+  - `input.reference_topics` (string, required): Comma-separated list of expected topics
+     * Required task output fields: completion, question, reference_topics
+     */
+    static topicAdherence(): EvaluatorWithConfig;
+    /**
+     * Detect toxic or harmful language
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to scan for toxic content
+  - `config.threshold` (float, optional): Detection threshold (default: 0.5)
+     * Required task output fields: text
+     */
+    static toxicityDetector(config?: ToxicityDetectorConfig): EvaluatorWithConfig;
+    /**
+     * Detect uncertainty in the text
+  
+  **Request Body:**
+  - `input.prompt` (string, required): The text to detect uncertainty in
+     * Required task output fields: prompt
+     */
+    static uncertaintyDetector(): EvaluatorWithConfig;
+    /**
+     * Count the number of words in text
+  
+  **Request Body:**
+  - `input.text` (string, required): The text to count words in
+     * Required task output fields: text
+     */
+    static wordCount(): EvaluatorWithConfig;
+    /**
+     * Calculate the ratio of words between two texts
+  
+  **Request Body:**
+  - `input.numerator_text` (string, required): The numerator text (will be divided by denominator)
+  - `input.denominator_text` (string, required): The denominator text (divides the numerator)
+     * Required task output fields: denominator_text, numerator_text
+     */
+    static wordCountRatio(): EvaluatorWithConfig;
+}
+//# sourceMappingURL=mbt-evaluators.d.ts.map

package/dist/src/lib/generated/evaluators/registry.d.ts

@@ -0,0 +1,12 @@
+export interface EvaluatorSchema {
+    slug: string;
+    requiredInputFields: string[];
+    optionalConfigFields: string[];
+    description?: string;
+}
+export type EvaluatorSlug = 'agent-efficiency' | 'agent-flow-quality' | 'agent-goal-accuracy' | 'agent-goal-completeness' | 'agent-tool-error-detector' | 'agent-tool-trajectory' | 'answer-completeness' | 'answer-correctness' | 'answer-relevancy' | 'char-count' | 'char-count-ratio' | 'context-relevance' | 'conversation-quality' | 'faithfulness' | 'html-comparison' | 'instruction-adherence' | 'intent-change' | 'json-validator' | 'perplexity' | 'pii-detector' | 'placeholder-regex' | 'profanity-detector' | 'prompt-injection' | 'prompt-perplexity' | 'regex-validator' | 'secrets-detector' | 'semantic-similarity' | 'sexism-detector' | 'sql-validator' | 'tone-detection' | 'topic-adherence' | 'toxicity-detector' | 'uncertainty-detector' | 'word-count' | 'word-count-ratio';
+export declare const EVALUATOR_SLUGS: EvaluatorSlug[];
+export declare const EVALUATOR_SCHEMAS: Record<EvaluatorSlug, EvaluatorSchema>;
+export declare function getEvaluatorSchema<S extends EvaluatorSlug>(slug: S): EvaluatorSchema;
+export declare function isValidEvaluatorSlug(slug: string): slug is EvaluatorSlug;
+//# sourceMappingURL=registry.d.ts.map