@orq-ai/evaluators 1.0.8 → 1.1.0-rc.2

package/README.md

@@ -10,15 +10,40 @@ npm install @orq-ai/evaluators
 
 ## Usage
 
+### String Contains Evaluator
+
+Check if the output contains the expected output (case-insensitive by default):
+
+```typescript
+import { stringContainsEvaluator } from "@orq-ai/evaluators";
+
+// Default: case-insensitive matching
+const evaluator = stringContainsEvaluator();
+
+// Case-sensitive matching
+const strictEvaluator = stringContainsEvaluator({
+  caseInsensitive: false
+});
+
+// Custom name
+const namedEvaluator = stringContainsEvaluator({
+  name: "contains-capital-city"
+});
+```
+
+The evaluator compares `output` against `data.expectedOutput` from the dataset and returns:
+- `value: 1.0` and `pass: true` if output contains expected
+- `value: 0.0` and `pass: false` otherwise
+
 ### Cosine Similarity Evaluator
 
 Compare semantic similarity between output and expected text using OpenAI embeddings:
 
 ```typescript
-import { 
-  cosineSimilarityEvaluator, 
+import {
+  cosineSimilarityEvaluator,
   cosineSimilarityThresholdEvaluator,
-  simpleCosineSimilarity 
+  simpleCosineSimilarity
 } from "@orq-ai/evaluators";
 
 // Simple usage - returns similarity score (0-1)

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./lib/cosine-similarity-evaluator.js";
+export * from "./lib/string-contains-evaluator.js";
 export * from "./lib/vector-utils.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAC;AACrD,cAAc,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAC;AACrD,cAAc,oCAAoC,CAAC;AACnD,cAAc,uBAAuB,CAAC"}

package/dist/index.js

@@ -1,2 +1,3 @@
 export * from "./lib/cosine-similarity-evaluator.js";
+export * from "./lib/string-contains-evaluator.js";
 export * from "./lib/vector-utils.js";

package/dist/lib/string-contains-evaluator.d.ts

@@ -0,0 +1,30 @@
+import type { Evaluator } from "@orq-ai/evaluatorq";
+/**
+ * Configuration options for the string contains evaluator
+ */
+export interface StringContainsConfig {
+    /**
+     * Whether the comparison should be case-insensitive
+     * @default true
+     */
+    caseInsensitive?: boolean;
+    /**
+     * Optional name for the evaluator
+     * @default "string-contains"
+     */
+    name?: string;
+}
+/**
+ * Creates an evaluator that checks if the output contains the expected output.
+ * Uses the data.expectedOutput from the dataset to compare against.
+ *
+ * @example
+ * ```typescript
+ * const evaluator = stringContainsEvaluator();
+ *
+ * // With case-sensitive matching
+ * const strictEvaluator = stringContainsEvaluator({ caseInsensitive: false });
+ * ```
+ */
+export declare function stringContainsEvaluator(config?: StringContainsConfig): Evaluator;
+//# sourceMappingURL=string-contains-evaluator.d.ts.map

package/dist/lib/string-contains-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"string-contains-evaluator.d.ts","sourceRoot":"","sources":["../../src/lib/string-contains-evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,GAAE,oBAAyB,GAChC,SAAS,CAiCX"}

package/dist/lib/string-contains-evaluator.js

@@ -0,0 +1,41 @@
+/**
+ * Creates an evaluator that checks if the output contains the expected output.
+ * Uses the data.expectedOutput from the dataset to compare against.
+ *
+ * @example
+ * ```typescript
+ * const evaluator = stringContainsEvaluator();
+ *
+ * // With case-sensitive matching
+ * const strictEvaluator = stringContainsEvaluator({ caseInsensitive: false });
+ * ```
+ */
+export function stringContainsEvaluator(config = {}) {
+    const { caseInsensitive = true, name = "string-contains" } = config;
+    return {
+        name,
+        scorer: async ({ data, output }) => {
+            const expected = String(data.expectedOutput ?? "");
+            const actual = String(output ?? "");
+            if (!expected) {
+                return {
+                    value: 0,
+                    pass: false,
+                    explanation: "No expected output defined",
+                };
+            }
+            const expectedNormalized = caseInsensitive
+                ? expected.toLowerCase()
+                : expected;
+            const actualNormalized = caseInsensitive ? actual.toLowerCase() : actual;
+            const contains = actualNormalized.includes(expectedNormalized);
+            return {
+                value: contains ? 1.0 : 0.0,
+                pass: contains,
+                explanation: contains
+                    ? `Output contains "${expected}"`
+                    : `Expected "${expected}" not found in: "${actual.substring(0, 100)}${actual.length > 100 ? "..." : ""}"`,
+            };
+        },
+    };
+}