@uipath/flow-tool 1.201.0-preview.131 → 1.201.0-preview.133

package/dist/services/flow-eval-schema/deterministic-evaluators.d.ts

@@ -0,0 +1,110 @@
+export declare const DETERMINISTIC_EVALUATOR_TYPES: {
+    readonly "uipath-contains": {
+        readonly name: "Contains Text";
+        readonly description: "Check if output contains specific text";
+        readonly category: "deterministic";
+        readonly shortDescription: "Passes when output contains given text";
+        readonly configFields: readonly [{
+            readonly key: "targetOutputKey";
+            readonly label: "Target Output Key";
+            readonly type: "targetOutputKey";
+            readonly required: false;
+            readonly description: "The key in the output to evaluate";
+        }, {
+            readonly key: "caseSensitive";
+            readonly label: "Case Sensitive";
+            readonly type: "boolean";
+            readonly required: false;
+            readonly default: false;
+            readonly description: "If true, the comparison is case-sensitive";
+        }, {
+            readonly key: "negated";
+            readonly type: "boolean";
+            readonly required: false;
+            readonly default: false;
+            readonly description: "If true, the evaluation passes when the match is not found";
+            readonly label: "Negated (should NOT contain)";
+        }];
+        readonly criteriaFields: readonly [{
+            readonly key: "searchText";
+            readonly label: "Search Text";
+            readonly type: "string";
+            readonly required: false;
+            readonly minLength: 1;
+            readonly description: "The text to search for in the output";
+        }];
+        readonly examples: {
+            readonly criteria: {
+                readonly searchText: "loan approved";
+            };
+        };
+    };
+    readonly "uipath-exact-match": {
+        readonly name: "Exact Match";
+        readonly description: "Check if output exactly matches expected value";
+        readonly category: "deterministic";
+        readonly shortDescription: "Passes when output is an exact match";
+        readonly configFields: readonly [{
+            readonly key: "targetOutputKey";
+            readonly label: "Target Output Key";
+            readonly type: "targetOutputKey";
+            readonly required: false;
+            readonly description: "The key in the output to evaluate";
+        }, {
+            readonly key: "caseSensitive";
+            readonly label: "Case Sensitive";
+            readonly type: "boolean";
+            readonly required: false;
+            readonly default: false;
+            readonly description: "If true, the comparison is case-sensitive";
+        }, {
+            readonly key: "negated";
+            readonly label: "Negated";
+            readonly type: "boolean";
+            readonly required: false;
+            readonly default: false;
+            readonly description: "If true, the evaluation passes when the match is not found";
+        }];
+        readonly criteriaFields: readonly [{
+            readonly key: "expectedOutput";
+            readonly label: "Expected Output";
+            readonly type: "object";
+            readonly required: false;
+            readonly description: "The expected output value for this evaluator";
+        }];
+        readonly examples: {
+            readonly criteria: {
+                readonly expectedOutput: {
+                    readonly value: "Application approved";
+                };
+            };
+        };
+    };
+    readonly "uipath-json-similarity": {
+        readonly name: "JSON Similarity";
+        readonly description: "Compare JSON output similarity";
+        readonly category: "deterministic";
+        readonly shortDescription: "Passes when JSON structures are similar";
+        readonly configFields: readonly [{
+            readonly key: "targetOutputKey";
+            readonly label: "Target Output Key";
+            readonly type: "targetOutputKey";
+            readonly required: false;
+            readonly description: "The key in the output to evaluate";
+        }];
+        readonly criteriaFields: readonly [{
+            readonly key: "expectedOutput";
+            readonly label: "Expected Output";
+            readonly type: "object";
+            readonly required: false;
+            readonly description: "The expected output value for this evaluator";
+        }];
+        readonly examples: {
+            readonly criteria: {
+                readonly expectedOutput: {
+                    readonly status: "ok";
+                };
+            };
+        };
+    };
+};

package/dist/services/flow-eval-schema/evaluator-fields.d.ts

@@ -0,0 +1,66 @@
+import type { EvaluatorFieldDefinition } from "./types.js";
+export declare const targetOutputKeyConfigField: {
+    readonly key: "targetOutputKey";
+    readonly label: "Target Output Key";
+    readonly type: "targetOutputKey";
+    readonly required: false;
+    readonly description: "The key in the output to evaluate";
+};
+export declare const caseSensitiveConfigField: {
+    readonly key: "caseSensitive";
+    readonly label: "Case Sensitive";
+    readonly type: "boolean";
+    readonly required: false;
+    readonly default: false;
+    readonly description: "If true, the comparison is case-sensitive";
+};
+export declare const negatedConfigField: {
+    readonly key: "negated";
+    readonly label: "Negated";
+    readonly type: "boolean";
+    readonly required: false;
+    readonly default: false;
+    readonly description: "If true, the evaluation passes when the match is not found";
+};
+export declare const expectedOutputCriteriaField: {
+    readonly key: "expectedOutput";
+    readonly label: "Expected Output";
+    readonly type: "object";
+    readonly required: false;
+    readonly description: "The expected output value for this evaluator";
+};
+export declare const llmBaseConfigFields: readonly [{
+    readonly key: "model";
+    readonly label: "Model";
+    readonly type: "llmModel";
+    readonly required: true;
+    readonly description: "The LLM model to use for judging";
+}, {
+    readonly key: "prompt";
+    readonly label: "Evaluation Prompt";
+    readonly type: "string";
+    readonly required: true;
+    readonly default: "As an expert evaluator, analyze the semantic similarity of these JSON contents to determine a score from 0-100. Focus on comparing the meaning and contextual equivalence of corresponding fields, accounting for alternative valid expressions, synonyms, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nExpectedOutput:\n{{ExpectedOutput}}\n----\nActualOutput:\n{{ActualOutput}}\n";
+    readonly description: "Custom prompt template for the LLM judge";
+}, {
+    readonly key: "temperature";
+    readonly label: "Temperature";
+    readonly type: "number";
+    readonly required: true;
+    readonly default: 0;
+    readonly min: 0;
+    readonly max: 2;
+    readonly step: 0.1;
+    readonly description: "Temperature for LLM generation";
+}, {
+    readonly key: "maxTokens";
+    readonly label: "Max Tokens";
+    readonly type: "number";
+    readonly required: true;
+    readonly default: 8192;
+    readonly min: 0;
+    readonly max: 16384;
+    readonly step: 256;
+    readonly description: "Maximum tokens for LLM response";
+}];
+export declare const llmTrajectoryBaseConfigFields: readonly EvaluatorFieldDefinition[];

package/dist/services/flow-eval-schema/helpers.d.ts

@@ -0,0 +1,9 @@
+import { type EvaluatorTypeId } from "./registry.js";
+import type { EvaluatorTypeDefinition } from "./types.js";
+export declare function evaluatorTypeAliases(): string[];
+export declare function isEvaluatorTypeId(value: string): value is EvaluatorTypeId;
+export declare function getEvaluatorTypeDefinition(typeId: string): EvaluatorTypeDefinition | null;
+export declare function resolveEvaluatorType(type: string): EvaluatorTypeId;
+export declare function getDefaultConfigForEvaluatorType(typeId: EvaluatorTypeId): Record<string, unknown>;
+export declare function evaluatorTypeHasConfigField(typeId: string, fieldKey: string): boolean;
+export declare function evaluatorCanUseSharedExpectedOutput(typeId: string): boolean;

package/dist/services/flow-eval-schema/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./helpers.js";
+export * from "./prompts.js";
+export * from "./registry.js";
+export * from "./types.js";

package/dist/services/flow-eval-schema/llm-judge-evaluators.d.ts

@@ -0,0 +1,174 @@
+export declare const LLM_JUDGE_EVALUATOR_TYPES: {
+    readonly "uipath-llm-judge-output-semantic-similarity": {
+        readonly name: "LLM Judge Output";
+        readonly description: "Use LLM to judge semantic similarity of outputs";
+        readonly category: "llm-judge";
+        readonly shortDescription: "AI judges if output semantically matches expected";
+        readonly configFields: readonly [{
+            readonly key: "model";
+            readonly label: "Model";
+            readonly type: "llmModel";
+            readonly required: true;
+            readonly description: "The LLM model to use for judging";
+        }, {
+            readonly key: "prompt";
+            readonly label: "Evaluation Prompt";
+            readonly type: "string";
+            readonly required: true;
+            readonly default: "As an expert evaluator, analyze the semantic similarity of these JSON contents to determine a score from 0-100. Focus on comparing the meaning and contextual equivalence of corresponding fields, accounting for alternative valid expressions, synonyms, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nExpectedOutput:\n{{ExpectedOutput}}\n----\nActualOutput:\n{{ActualOutput}}\n";
+            readonly description: "Custom prompt template for the LLM judge";
+        }, {
+            readonly key: "temperature";
+            readonly label: "Temperature";
+            readonly type: "number";
+            readonly required: true;
+            readonly default: 0;
+            readonly min: 0;
+            readonly max: 2;
+            readonly step: 0.1;
+            readonly description: "Temperature for LLM generation";
+        }, {
+            readonly key: "maxTokens";
+            readonly label: "Max Tokens";
+            readonly type: "number";
+            readonly required: true;
+            readonly default: 8192;
+            readonly min: 0;
+            readonly max: 16384;
+            readonly step: 256;
+            readonly description: "Maximum tokens for LLM response";
+        }, {
+            readonly key: "targetOutputKey";
+            readonly label: "Target Output Key";
+            readonly type: "targetOutputKey";
+            readonly required: false;
+            readonly description: "The key in the output to evaluate";
+        }];
+        readonly criteriaFields: readonly [{
+            readonly key: "expectedOutput";
+            readonly label: "Expected Output";
+            readonly type: "object";
+            readonly required: false;
+            readonly description: "The expected output value for this evaluator";
+        }];
+        readonly examples: {
+            readonly config: {
+                readonly model: "gpt-4o";
+                readonly temperature: 0;
+            };
+            readonly criteria: {
+                readonly expectedOutput: {
+                    readonly text: "The loan has been approved for the requested amount.";
+                };
+            };
+        };
+    };
+    readonly "uipath-llm-judge-output-strict-json-similarity": {
+        readonly name: "LLM Judge Strict JSON";
+        readonly description: "Use LLM to judge strict JSON similarity";
+        readonly category: "llm-judge";
+        readonly shortDescription: "AI judges strict JSON structural match";
+        readonly configFields: readonly [{
+            readonly key: "model";
+            readonly label: "Model";
+            readonly type: "llmModel";
+            readonly required: true;
+            readonly description: "The LLM model to use for judging";
+        }, {
+            readonly key: "prompt";
+            readonly label: "Evaluation Prompt";
+            readonly type: "string";
+            readonly required: true;
+            readonly default: "As an expert evaluator, analyze the semantic similarity of these JSON contents to determine a score from 0-100. Focus on comparing the meaning and contextual equivalence of corresponding fields, accounting for alternative valid expressions, synonyms, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nExpectedOutput:\n{{ExpectedOutput}}\n----\nActualOutput:\n{{ActualOutput}}\n";
+            readonly description: "Custom prompt template for the LLM judge";
+        }, {
+            readonly key: "temperature";
+            readonly label: "Temperature";
+            readonly type: "number";
+            readonly required: true;
+            readonly default: 0;
+            readonly min: 0;
+            readonly max: 2;
+            readonly step: 0.1;
+            readonly description: "Temperature for LLM generation";
+        }, {
+            readonly key: "maxTokens";
+            readonly label: "Max Tokens";
+            readonly type: "number";
+            readonly required: true;
+            readonly default: 8192;
+            readonly min: 0;
+            readonly max: 16384;
+            readonly step: 256;
+            readonly description: "Maximum tokens for LLM response";
+        }, {
+            readonly key: "targetOutputKey";
+            readonly label: "Target Output Key";
+            readonly type: "targetOutputKey";
+            readonly required: false;
+            readonly description: "The key in the output to evaluate";
+        }];
+        readonly criteriaFields: readonly [{
+            readonly key: "expectedOutput";
+            readonly label: "Expected Output";
+            readonly type: "object";
+            readonly required: false;
+            readonly description: "The expected output value for this evaluator";
+        }];
+        readonly examples: {
+            readonly config: {
+                readonly model: "gpt-4o";
+                readonly temperature: 0;
+            };
+            readonly criteria: {
+                readonly expectedOutput: {
+                    readonly result: "success";
+                };
+            };
+        };
+    };
+    readonly "uipath-llm-judge-trajectory-similarity": {
+        readonly name: "LLM Judge Trajectory";
+        readonly description: "Use LLM to judge agent trajectory and behavior";
+        readonly category: "llm-judge";
+        readonly shortDescription: "AI judges if agent followed expected behavior";
+        readonly configFields: readonly import("./types.js").EvaluatorFieldDefinition[];
+        readonly criteriaFields: readonly [{
+            readonly key: "expectedAgentBehavior";
+            readonly label: "Expected Agent Behavior";
+            readonly type: "string";
+            readonly required: false;
+            readonly description: "Description of expected agent behavior and actions";
+        }];
+        readonly examples: {
+            readonly config: {
+                readonly model: "gpt-4o";
+            };
+            readonly criteria: {
+                readonly expectedAgentBehavior: "Agent checks eligibility, then calls approval API";
+            };
+        };
+    };
+    readonly "uipath-llm-judge-trajectory-simulation": {
+        readonly name: "LLM Judge Simulation";
+        readonly description: "Use LLM to judge simulation trajectory";
+        readonly category: "llm-judge";
+        readonly shortDescription: "AI judges simulated agent execution path";
+        readonly configFields: readonly import("./types.js").EvaluatorFieldDefinition[];
+        readonly criteriaFields: readonly [{
+            readonly key: "expectedAgentBehavior";
+            readonly label: "Expected Agent Behavior";
+            readonly type: "string";
+            readonly required: false;
+            readonly description: "Description of expected agent behavior during simulation";
+        }];
+        readonly examples: {
+            readonly config: {
+                readonly model: "gpt-4o";
+            };
+            readonly criteria: {
+                readonly expectedAgentBehavior: "Agent simulates the user journey through checkout flow";
+            };
+        };
+    };
+};

package/dist/services/flow-eval-schema/prompts.d.ts

@@ -0,0 +1,2 @@
+export declare const DEFAULT_EVALUATOR_PROMPT = "As an expert evaluator, analyze the semantic similarity of these JSON contents to determine a score from 0-100. Focus on comparing the meaning and contextual equivalence of corresponding fields, accounting for alternative valid expressions, synonyms, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nExpectedOutput:\n{{ExpectedOutput}}\n----\nActualOutput:\n{{ActualOutput}}\n";
+export declare const DEFAULT_TRAJECTORY_EVALUATOR_PROMPT = "As an expert evaluator, determine how well the agent did on a scale of 0-100. Focus on if the simulation was successful and if the agent behaved according to the expected output accounting for alternative valid expressions, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nUserOrSyntheticInputGivenToAgent:\n{{UserOrSyntheticInput}}\n----\nSimulationInstructions:\n{{SimulationInstructions}}\n----\nExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n----\nAgentRunHistory:\n{{AgentRunHistory}}\n";