@arizeai/phoenix-evals 0.8.0 → 0.10.0

package/dist/esm/types/evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB,CAClC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACrE,SAAQ,aAAa;IACrB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,YAAY,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,CAAC;CAC3C;AAED,MAAM,MAAM,sBAAsB,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC3E,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,iCAAiC,CAChD,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1C,SAAQ,oBAAoB,EAAE,sBAAsB,CAAC,UAAU,CAAC;IAChE;;OAEG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB,CACjC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3C,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}

package/dist/esm/types/otel.d.ts

@@ -1,4 +1,4 @@
-import { Tracer } from "@opentelemetry/api";
+import type { Tracer } from "@opentelemetry/api";
 export type TelemetryConfig = {
     /**
      * Whether OpenTelemetry is enabled on the call.

package/dist/esm/types/otel.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,CAAC"}
+{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../../../src/types/otel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,CAAC"}

package/dist/esm/utils/bindEvaluator.d.ts

@@ -1,5 +1,5 @@
-import { EvaluatorBase } from "../core/EvaluatorBase.js";
-import { ObjectMapping } from "../types/data.js";
+import type { EvaluatorBase } from "../core/EvaluatorBase.js";
+import type { ObjectMapping } from "../types/data.js";
 /**
  * Context for binding an evaluator with input mapping configuration.
  *

package/dist/esm/utils/bindEvaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bindEvaluator.d.ts","sourceRoot":"","sources":["../../../src/utils/bindEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,MAAM,MAAM,cAAc,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI;IACvE;;;;;;;;;;;;OAYG;IACH,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACzC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0JG;AACH,wBAAgB,aAAa,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACtE,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,EACpC,OAAO,EAAE,cAAc,CAAC,UAAU,CAAC,GAClC,aAAa,CAAC,UAAU,CAAC,CAM3B"}
+{"version":3,"file":"bindEvaluator.d.ts","sourceRoot":"","sources":["../../../src/utils/bindEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,MAAM,MAAM,cAAc,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI;IACvE;;;;;;;;;;;;OAYG;IACH,YAAY,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACzC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0JG;AACH,wBAAgB,aAAa,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACtE,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,EACpC,OAAO,EAAE,cAAc,CAAC,UAAU,CAAC,GAClC,aAAa,CAAC,UAAU,CAAC,CAM3B"}

package/dist/esm/utils/objectMappingUtils.d.ts

@@ -1,4 +1,4 @@
-import { ObjectMapping } from "../types/data.js";
+import type { ObjectMapping } from "../types/data.js";
 /**
  * Remaps an object by applying field mappings while preserving original data.
  *

package/dist/esm/utils/objectMappingUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"objectMappingUtils.d.ts","sourceRoot":"","sources":["../../../src/utils/objectMappingUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAe,MAAM,eAAe,CAAC;AAI3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkKG;AACH,wBAAgB,WAAW,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClE,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC/B,QAAQ,CAUV"}
+{"version":3,"file":"objectMappingUtils.d.ts","sourceRoot":"","sources":["../../../src/utils/objectMappingUtils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAe,MAAM,eAAe,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkKG;AACH,wBAAgB,WAAW,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClE,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC/B,QAAQ,CAUV"}

package/dist/esm/utils/objectMappingUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"objectMappingUtils.js","sourceRoot":"","sources":["../../../src/utils/objectMappingUtils.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkKG;AACH,MAAM,UAAU,WAAW,CACzB,IAAc,EACd,OAAgC;IAEhC,OAAO;QACL,GAAG,IAAI;QACP,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YAC5C,GAAG;YACH,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC;SAClC,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,oBAAoB,CAC3B,IAAc,EACd,WAAkC;IAElC,OAAO,OAAO,WAAW,KAAK,UAAU;QACtC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC;QACnB,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC/D,CAAC"}
+{"version":3,"file":"objectMappingUtils.js","sourceRoot":"","sources":["../../../src/utils/objectMappingUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAIzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkKG;AACH,MAAM,UAAU,WAAW,CACzB,IAAc,EACd,OAAgC;IAEhC,OAAO;QACL,GAAG,IAAI;QACP,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YAC5C,GAAG;YACH,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC;SAClC,CAAC,CACH;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,oBAAoB,CAC3B,IAAc,EACd,WAAkC;IAElC,OAAO,OAAO,WAAW,KAAK,UAAU;QACtC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC;QACnB,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC/D,CAAC"}

package/dist/src/__generated__/default_templates/CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts

@@ -0,0 +1,3 @@
+import type { ClassificationEvaluatorConfig } from "../types";
+export declare const CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig;
+//# sourceMappingURL=CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

package/dist/src/__generated__/default_templates/CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,2CAA2C,EAAE,6BAuDzD,CAAC"}

package/dist/src/__generated__/default_templates/CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -0,0 +1,61 @@
+"use strict";
+// This file is generated. Do not edit by hand.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
+exports.CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG = {
+    name: "conciseness",
+    description: "Evaluate whether model outputs are concise and free of unnecessary content.",
+    optimizationDirection: "MAXIMIZE",
+    template: [
+        {
+            role: "user",
+            content: `
+You are an expert evaluator assessing the conciseness of model outputs. Your task is to determine whether a response uses the minimum number of words necessary to fully answer the question.
+
+<rubric>
+
+CONCISE - The response:
+
+- Contains only the exact information requested
+- Uses the minimum number of words necessary to convey the complete answer
+- Omits pleasantries, hedging language, and unnecessary context
+- Excludes meta-commentary about the answer or the model's capabilities
+- Avoids redundant information or restatements
+- Does not include explanations unless explicitly requested
+
+
+VERBOSE - The response contains any of:
+
+- Unnecessary pleasantries, greetings, or filler phrases (e.g., "Great question!", "Sure!", "I'd be happy to help")
+- Hedging language or excessive qualifiers (e.g., "It's worth noting that...", "It's important to understand that...")
+- Meta-commentary about the response itself or the model's capabilities
+- Redundant restatements of the same information
+- Unsolicited explanations, context, or caveats beyond what was asked
+- Unnecessary formatting, bullet points, or structure for simple answers
+
+</rubric>
+
+<data>
+
+<input>
+{{input}}
+</input>
+
+<output>
+{{output}}
+</output>
+
+</data>
+
+Evaluate only the conciseness of the response. Do not assess correctness, helpfulness, or quality of information. Focus solely on whether the response uses more words than necessary to answer the question.
+
+Is the output concise or verbose?
+`,
+        },
+    ],
+    choices: {
+        "concise": 1,
+        "verbose": 0
+    },
+};
+//# sourceMappingURL=CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map

package/dist/src/__generated__/default_templates/CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/CONCISENESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,2CAA2C,GAAkC;IACxF,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,6EAA6E;IAC1F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyCd;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;KACb;CACA,CAAC"}

package/dist/src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,2CAA2C,EAAE,6BA8CzD,CAAC"}
+{"version":3,"file":"CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,2CAA2C,EAAE,6BAsDzD,CAAC"}

package/dist/src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
 exports.CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG = {
     name: "correctness",
-    description: "Assess factual accuracy and completeness of model outputs.",
+    description: "Assess general correctness and completeness of model outputs.",
     optimizationDirection: "MAXIMIZE",
     template: [
         {
@@ -13,29 +13,37 @@ exports.CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG = {
 You are an expert evaluator labeling model outputs for correctness. Your task is to assign a classification based on the following criteria:
 
 <rubric>
+
 CORRECT - The response:
+
 - Provides accurate and complete information with no factual errors
 - Addresses all parts of the question
 - Is logically consistent with no contradictions
 - Uses precise, domain-appropriate terminology
 - Avoids ambiguous or misleading language
 
+
 INCORRECT - The response contains any of:
+
 - Factual errors or inaccuracies
 - Incomplete or partial answers
 - Misleading or ambiguous statements
 - Incorrect terminology
 - Logical inconsistencies
 - Missing key information
+
 </rubric>
 
 <data>
+
 <input>
 {{input}}
 </input>
+
 <output>
 {{output}}
 </output>
+
 </data>
 
 Carefully read the input and output and check for factual accuracy and completeness. Focus on correctness of information rather than verboseness or style.

package/dist/src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -1 +1 @@
-{"version":3,"file":"CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,2CAA2C,GAAkC;IACxF,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,4DAA4D;IACzE,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCd;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}
+{"version":3,"file":"CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/CORRECTNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,2CAA2C,GAAkC;IACxF,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,+DAA+D;IAC5E,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwCd;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,kDAAkD,EAAE,6BA2BhE,CAAC"}
+{"version":3,"file":"DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,kDAAkD,EAAE,6BAsChE,CAAC"}

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -4,24 +4,35 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
 exports.DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG = {
     name: "document_relevance",
-    description: "A specialized evaluator for determining document relevance to a given question.",
+    description: "For determining if a document is relevant to a given question.",
     optimizationDirection: "MAXIMIZE",
     template: [
         {
             role: "user",
             content: `
-You are comparing a document to a question and trying to determine if the document text contains information relevant to answering the question. Here is the data:
+You are comparing a document to a question and trying to determine
+if the document text contains information relevant to answering the
+question. Here is the data:
 
 <data>
+
 <question>
 {{input}}
 </question>
+
 <document_text>
 {{documentText}}
 </document_text>
+
 </data>
 
-Compare the question above to the document text. You must determine whether the document text contains information that can answer the question. Please focus on whether the very specific question can be answered by the information in the document text. Your response must be either "relevant" or "unrelated". "unrelated" means that the document text does not contain an answer to the question. "relevant" means the document text contains an answer to the question.
+Compare the question above to the document text. You must determine
+whether the document text contains information that can answer the
+question. Please focus on whether the very specific question can be
+answered by the information in the document text. Your response must be
+either "relevant" or "unrelated". "unrelated" means that the document
+text does not contain an answer to the question. "relevant" means the
+document text contains an answer to the question.
 `,
         },
     ],

package/dist/src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -1 +1 @@
-{"version":3,"file":"DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,kDAAkD,GAAkC;IAC/F,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,iFAAiF;IAC9F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;CAad;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}
+{"version":3,"file":"DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/DOCUMENT_RELEVANCE_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,kDAAkD,GAAkC;IAC/F,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,gEAAgE;IAC7E,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;CAwBd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}

package/dist/src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,4CAA4C,EAAE,6BA8B1D,CAAC"}
+{"version":3,"file":"FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,4CAA4C,EAAE,6BA4C1D,CAAC"}

package/dist/src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -4,24 +4,38 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
 exports.FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG = {
     name: "faithfulness",
-    description: "A specialized evaluator for detecting faithfulness in grounded LLM responses.",
+    description: "For determining if a response is faithful to the context.",
     optimizationDirection: "MAXIMIZE",
     template: [
         {
             role: "user",
             content: `
-In this task, you will be presented with a query, some context and a response. The response is generated to the question based on the context. The response may contain false information. You must use the context to determine if the response to the question contains false information, if the response is unfaithful to the facts. Your objective is to determine whether the response text contains factual information and is faithful to the context. An 'unfaithful' response refers to a response that is not based on the context or assumes information that is not available in the context. Your response should be a single word: either 'faithful' or 'unfaithful', and it should not include any other text or characters. 'unfaithful' indicates that the response provides factually inaccurate information to the query based on the context. 'faithful' indicates that the response to the question is correct relative to the context, and does not contain made up information. Please read the query and context carefully before determining your response.
+In this task, you will be presented with a query, some context and a response. The response is generated to the question based on the context. The response may contain false information. You must use the context to determine if the response to the question contains false information, if the response is unfaithful to the facts. 
+
+Your objective is to determine whether the response text contains factual information and is faithful to the context. An 'unfaithful' response refers to a response that is not based on the context or assumes information that is not available in the context. 
+
+Your response should be a single word: either 'faithful' or 'unfaithful', and it should not include any other text or characters. 
+
+'unfaithful' indicates that the response provides factually inaccurate information to the query based on the context. 
+
+'faithful' indicates that the response to the question is correct relative to the context, and does not contain made up information. 
+
+Please read the query and context carefully before determining your response.
 
 <data>
+
 <query>
 {{input}}
 </query>
+
 <context>
 {{context}}
 </context>
+
 <response>
 {{output}}
 </response>
+
 </data>
 
 Is the response above faithful or unfaithful based on the query and context?

package/dist/src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -1 +1 @@
-{"version":3,"file":"FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,4CAA4C,GAAkC;IACzF,IAAI,EAAE,cAAc;IACpB,WAAW,EAAE,+EAA+E;IAC5F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;CAgBd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB;CACA,CAAC"}
+{"version":3,"file":"FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,4CAA4C,GAAkC;IACzF,IAAI,EAAE,cAAc;IACpB,WAAW,EAAE,2DAA2D;IACxE,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Bd;SACI;KACF;IACD,OAAO,EAAE;QACT,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;KAChB;CACA,CAAC"}

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,6CAA6C,EAAE,6BA8B3D,CAAC"}
+{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,6CAA6C,EAAE,6BA4C3D,CAAC"}

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -10,18 +10,32 @@ exports.HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG = {
         {
             role: "user",
             content: `
-In this task, you will be presented with a query, some context and a response. The response is generated to the question based on the context. The response may contain false information. You must use the context to determine if the response to the question contains false information, if the response is a hallucination of facts. Your objective is to determine whether the response text contains factual information and is not a hallucination. A 'hallucination' refers to a response that is not based on the context or assumes information that is not available in the context. Your response should be a single word: either 'factual' or 'hallucinated', and it should not include any other text or characters. 'hallucinated' indicates that the response provides factually inaccurate information to the query based on the context. 'factual' indicates that the response to the question is correct relative to the context, and does not contain made up information. Please read the query and context carefully before determining your response.
+In this task, you will be presented with a query, some context and a response. The response is generated to the question based on the context. The response may contain false information. You must use the context to determine if the response to the question contains false information, if the response is hallucinated. 
+
+Your objective is to determine whether the response text contains factual information and is factual relative to the context. An 'hallucinated' response refers to a response that is not based on the context or assumes information that is not available in the context. 
+
+Your response should be a single word: either 'factual' or 'hallucinated', and it should not include any other text or characters. 
+
+'hallucinated' indicates that the response provides factually inaccurate information to the query based on the context. 
+
+'factual' indicates that the response to the question is correct relative to the context, and does not contain made up information. 
+
+Please read the query and context carefully before determining your response.
 
 <data>
+
 <query>
 {{input}}
 </query>
+
 <context>
 {{context}}
 </context>
+
 <response>
 {{output}}
 </response>
+
 </data>
 
 Is the response above factual or hallucinated based on the query and context?

package/dist/src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -1 +1 @@
-{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,6CAA6C,GAAkC;IAC1F,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,iFAAiF;IAC9F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;CAgBd;SACI;KACF;IACD,OAAO,EAAE;QACT,cAAc,EAAE,CAAC;QACjB,SAAS,EAAE,CAAC;KACb;CACA,CAAC"}
+{"version":3,"file":"HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/HALLUCINATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,6CAA6C,GAAkC;IAC1F,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,iFAAiF;IAC9F,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Bd;SACI;KACF;IACD,OAAO,EAAE;QACT,cAAc,EAAE,CAAC;QACjB,SAAS,EAAE,CAAC;KACb;CACA,CAAC"}

package/dist/src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,+CAA+C,EAAE,6BA2B7D,CAAC"}
+{"version":3,"file":"TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,+CAA+C,EAAE,6BA6E7D,CAAC"}

package/dist/src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -4,23 +4,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
 exports.TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG = {
     name: "tool_invocation",
-    description: "For determining if a tool was invoked correctly with proper arguments, formatting, and safe content. Requires conversation context, available tool schemas, and the LLM's tool invocation(s).",
+    description: "For determining if a tool was invoked correctly with proper arguments, formatting, and safe content.",
     optimizationDirection: "MAXIMIZE",
     template: [
         {
             role: "user",
             content: `
 You are an impartial judge evaluating an LLM's tool-calling behavior, specifically whether the LLM invoked a tool (or tools) correctly with valid arguments and proper formatting.
-Your task: Determine whether the LLM's tool invocation(s) were correct or incorrect based on: - The full conversation context (including all previous turns, not just the most recent message) - The available tool schemas - The LLM's tool invocation(s) with arguments
-IMPORTANT - Tool Invocation vs. Tool Selection: - You are ONLY evaluating the tool invocation, not the tool selection. - If the tool selection is incorrect or not relevant to the user's query, but the tool invocation is correct, return "correct". - If the tool selection is correct but the tool invocation is incorrect, return "incorrect".
-IMPORTANT - Multi-Tool Invocations: - The LLM may invoke MULTIPLE tools in a single response. This is valid and expected for complex requests. - When multiple tools are invoked, evaluate EACH tool invocation independently. - Return "correct" only if ALL tool invocations are correct. - Return "incorrect" if ANY tool invocation has an error.
-IMPORTANT - Conversation Context: - Read the entire conversation history carefully, not just the final user message. - Argument values may need to be extracted from EARLIER turns in the conversation (e.g., user mentions a location, date, or quantity in a previous message). - The LLM should use context from the full conversation to populate argument values correctly.
-Criteria Return "correct" only when ALL of the following are true for EVERY tool invocation: - JSON is properly structured (if applicable). - All required fields/parameters are present. - No hallucinated or nonexistent fields (all fields exist in the tool schema). - Argument values match the user's intent from the conversation context (correct types, realistic values). - No unsafe content (e.g., PII like SSNs, credit card numbers, passwords) in arguments.
-Return "incorrect" if ANY of the following are true for ANY tool invocation: - The invocation contains hallucinated or nonexistent fields not in the schema. - Required fields/parameters are missing. - JSON is improperly formatted or malformed. - Argument values are incorrect, hallucinated, or do not match user intent from the conversation. - Arguments contain unsafe content (e.g., PII, sensitive data that should not be passed).
-Before providing your final judgment, explain your reasoning and consider: - How many tools were invoked? Evaluate each one. - Does each tool invocation match the schema for that tool? - Are all required parameters provided with appropriate values for each invocation? - Are there any extra fields that don't exist in the schema? - Looking at the FULL conversation: do the argument values accurately reflect what the user requested across all messages? - Is there any unsafe or sensitive content in any of the arguments? - Check that you are not evaluating the tool selection, only the tool invocation.
-<data> <context> {{input}} </context>
-<available_tools> {{availableTools}} </available_tools>
-<tool_invocation> {{toolSelection}} </tool_invocation> </data>
+
+Your task: Determine whether the LLM's tool invocation(s) were correct or incorrect based on:
+- The full conversation context (including all previous turns, not just the most recent message)
+- The available tool schemas
+- The LLM's tool invocation(s) with arguments
+
+IMPORTANT - Tool Invocation vs. Tool Selection:
+- You are ONLY evaluating the tool invocation, not the tool selection.
+- If the tool selection is incorrect or not relevant to the user's query, but the tool invocation is correct, return "correct".
+- If the tool selection is correct but the tool invocation is incorrect, return "incorrect".
+
+IMPORTANT - Multi-Tool Invocations:
+- The LLM may invoke MULTIPLE tools in a single response. This is valid and expected for complex requests.
+- When multiple tools are invoked, evaluate EACH tool invocation independently.
+- Return "correct" only if ALL tool invocations are correct.
+- Return "incorrect" if ANY tool invocation has an error.
+
+IMPORTANT - Conversation Context (input):
+- Read the entire conversation history carefully, not just the final user message.
+- Argument values may need to be extracted from EARLIER turns in the conversation (e.g., user mentions a location, date, or quantity in a previous message).
+- The LLM should use context from the full conversation to populate argument values correctly.
+
+Criteria
+Return "correct" only when ALL of the following are true for EVERY tool invocation:
+- JSON is properly structured (if applicable).
+- All required fields/parameters are present.
+- No hallucinated or nonexistent fields (all fields exist in the tool schema).
+- Argument values match the user's intent from the conversation context (correct types, realistic values).
+- No unsafe content (e.g., PII like SSNs, credit card numbers, passwords) in arguments.
+
+Return "incorrect" if ANY of the following are true for ANY tool invocation:
+- The invocation contains hallucinated or nonexistent fields not in the schema.
+- Required fields/parameters are missing.
+- JSON is improperly formatted or malformed.
+- Argument values are incorrect, hallucinated, or do not match user intent from the conversation.
+- Arguments contain unsafe content (e.g., PII, sensitive data that should not be passed).
+
+Before providing your final judgment, explain your reasoning and consider:
+- How many tools were invoked? Evaluate each one.
+- Does each tool invocation match the schema for that tool?
+- Are all required parameters provided with appropriate values for each invocation?
+- Are there any extra fields that don't exist in the schema?
+- Looking at the FULL input: do the argument values accurately reflect what the user requested across all messages?
+- Is there any unsafe or sensitive content in any of the arguments?
+- Check that you are not evaluating the tool selection, only the tool invocation.
+
+<data>
+<input>
+{{input}}
+</input>
+
+<available_tools>
+{{availableTools}}
+</available_tools>
+
+<output>
+{{toolSelection}}
+</output>
+</data>
+
 Given the above data, is the tool invocation correct or incorrect?
 `,
         },

package/dist/src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -1 +1 @@
-{"version":3,"file":"TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,+CAA+C,GAAkC;IAC5F,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,+LAA+L;IAC5M,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;CAad;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}
+{"version":3,"file":"TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_INVOCATION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,+CAA+C,GAAkC;IAC5F,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,sGAAsG;IACnH,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Dd;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}

package/dist/src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts

@@ -0,0 +1,3 @@
+import type { ClassificationEvaluatorConfig } from "../types";
+export declare const TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig;
+//# sourceMappingURL=TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

package/dist/src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,sDAAsD,EAAE,6BAmFpE,CAAC"}

package/dist/src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js

@@ -0,0 +1,89 @@
+"use strict";
+// This file is generated. Do not edit by hand.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG = void 0;
+exports.TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG = {
+    name: "tool_response_handling",
+    description: "For determining if an AI agent properly handled a tool's response, including error handling, data extraction, transformation, and safe information disclosure. Requires conversation context, the tool call(s), the tool result(s), and the agent's output.",
+    optimizationDirection: "MAXIMIZE",
+    template: [
+        {
+            role: "user",
+            content: `
+You are an impartial judge evaluating an AI agent's handling of a tool's response. Your task is to determine whether the agent correctly processed the tool result to produce an appropriate output.
+
+IMPORTANT - Scope of Evaluation:
+- You are ONLY evaluating how the agent handled the tool response, NOT whether the right tool was selected or whether the tool was invoked correctly.
+- This evaluation focuses on what happens AFTER the tool returns a result.
+
+IMPORTANT - Multi-Tool Handling:
+- The agent may make MULTIPLE tool calls in a single interaction. This is valid and expected.
+- When multiple tools are called, evaluate how the agent handled ALL tool results together.
+- Return "correct" only if the agent properly handled ALL tool results.
+- Return "incorrect" if the agent mishandled ANY tool result.
+
+IMPORTANT - Error Response Handling:
+- Tool results may contain errors (rate limits, timeouts, not found, invalid arguments, etc.).
+- The agent's output may include retries, follow-up tool calls, or a final response to the user.
+- Evaluate the ENTIRE handling sequence, not just the final message.
+- Appropriate error handling includes:
+  - Retrying on transient errors (rate limits, timeouts)
+  - Correcting arguments after invalid argument errors
+  - Informing the user appropriately when errors are not recoverable
+  - NOT making repeated identical calls that continue to fail
+
+Criteria for CORRECT handling:
+- Data is extracted accurately from the tool result (no hallucination of data that wasn't returned)
+- Dates, numbers, and structured fields are properly transformed and formatted
+- Results are accurately summarized to address the user's original query
+- Error responses are handled appropriately (retries for transient errors, corrections for invalid arguments)
+- No repeated identical calls after non-retryable errors
+- No disclosure of sensitive/internal information (database credentials, internal URLs, PII, API keys, etc.)
+- The agent's response actually uses the tool result rather than ignoring it
+
+Criteria for INCORRECT handling:
+- Hallucinated data: The output includes information not present in the tool result
+- Misinterpretation: The meaning of the tool result is misrepresented or reversed
+- Improper transformation: Dates, numbers, or structured data are incorrectly converted
+- Missing retry: Failed to retry on retryable errors (rate limits, timeouts)
+- Missing correction: Failed to correct arguments after invalid argument errors
+- Futile retries: Repeated identical calls that continue to fail
+- Information disclosure: Leaked sensitive information (credentials, internal URLs, PII)
+- Ignored results: The agent's response doesn't incorporate the tool result
+- Incomplete handling: Only some tool results are used when multiple tools were called
+
+Before providing your final judgment, explain your reasoning and consider:
+- Does the output accurately reflect what the tool returned?
+- Are there any fabricated details not in the tool result?
+- Were errors handled appropriately?
+- Is sensitive information properly protected?
+- Does the output actually address the user's query using the tool data?
+
+<data>
+<input>
+{{input}}
+</input>
+
+<tool_call>
+{{toolCall}}
+</tool_call>
+
+<tool_result>
+{{toolResult}}
+</tool_result>
+
+<output>
+{{output}}
+</output>
+</data>
+
+Given the above data, did the agent handle the tool response correctly or incorrectly?
+`,
+        },
+    ],
+    choices: {
+        "correct": 1,
+        "incorrect": 0
+    },
+};
+//# sourceMappingURL=TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js.map

package/dist/src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.js","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_RESPONSE_HANDLING_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":";AAAA,+CAA+C;;;AAIlC,QAAA,sDAAsD,GAAkC;IACnG,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,6PAA6P;IAC1Q,qBAAqB,EAAE,UAAU;IACjC,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqEd;SACI;KACF;IACD,OAAO,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,CAAC;KACf;CACA,CAAC"}

package/dist/src/__generated__/default_templates/TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,8CAA8C,EAAE,6BAwB5D,CAAC"}
+{"version":3,"file":"TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.d.ts","sourceRoot":"","sources":["../../../../src/__generated__/default_templates/TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,UAAU,CAAC;AAE9D,eAAO,MAAM,8CAA8C,EAAE,6BA4D5D,CAAC"}