@cogitator-ai/evals 0.6.1 → 0.8.0

package/README.md

@@ -17,7 +17,7 @@ Website docs: [Evals](https://cogitator.app/docs/evals), [Datasets](https://cogi
 ## Features
 
 - **EvalSuite** — Run datasets against agents or plain functions with configurable concurrency, timeouts, and retries
-- **4 Deterministic Metrics** — exactMatch, contains, regex, jsonSchema (Zod)
+- **7 Deterministic Metrics** - exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema (Zod)
 - **5 LLM-as-Judge Metrics** — faithfulness, relevance, coherence, helpfulness, custom llmMetric
 - **3 Statistical Metrics** — latency, cost, tokenUsage with full percentile breakdowns
 - **Custom Metrics** — `metric()` factory for anything domain-specific
@@ -152,25 +152,43 @@ All transformations return new `Dataset` instances — the original is never mut
 
 ### Deterministic
 
-Binary (0 or 1) metrics that compare output against expected values.
+Metrics that compare output against expected values using string operations or schema validation.
 
-| Metric       | Description                                | Requires `expected` |
-| ------------ | ------------------------------------------ | ------------------- |
-| `exactMatch` | Exact string match (case optional)         | Yes                 |
-| `contains`   | Output contains expected substring         | Yes                 |
-| `regex`      | Output matches a regex pattern             | No                  |
-| `jsonSchema` | Output is valid JSON matching a Zod schema | No                  |
+| Metric        | Description                                    | Requires `expected` |
+| ------------- | ---------------------------------------------- | ------------------- |
+| `exactMatch`  | Exact string match (case optional)             | Yes                 |
+| `contains`    | Output contains expected substring             | Yes                 |
+| `containsAll` | Fraction of configured keywords found          | No                  |
+| `containsAny` | Whether any configured keyword is found        | No                  |
+| `similarity`  | Levenshtein distance similarity score (0 to 1) | Yes                 |
+| `regex`       | Output matches a regex pattern                 | No                  |
+| `jsonSchema`  | Output is valid JSON matching a Zod schema     | No                  |
 
 ```typescript
-import { exactMatch, contains, regex, jsonSchema } from '@cogitator-ai/evals';
+import {
+  exactMatch,
+  contains,
+  containsAll,
+  containsAny,
+  similarity,
+  regex,
+  jsonSchema,
+  threshold,
+} from '@cogitator-ai/evals';
 import { z } from 'zod';
 
 const metrics = [
   exactMatch({ caseSensitive: true }),
   contains(),
+  containsAll(['security', 'privacy', 'compliance']),
+  containsAny(['approved', 'accepted']),
+  similarity(),
   regex(/\d{4}-\d{2}-\d{2}/),
   jsonSchema(z.object({ answer: z.string(), confidence: z.number() })),
 ];
+
+// Assert similarity is at least 0.8
+const assertions = [threshold('similarity', 0.8)];
 ```
 
 Every built-in per-case metric takes a `name` option. A suite refuses two metrics with the same name, because their scores would merge into one aggregate, so name the second of a kind:
@@ -182,6 +200,8 @@ const metrics = [
 ];
 ```
 
+`containsAll()` scores the fraction of its keywords found and reports any missing keywords in `details`. `containsAny()` scores 1 when at least one keyword is found and 0 otherwise. Keywords use substring matching, so `cat` matches `concatenate`. Both metrics are case-insensitive by default, accept `caseSensitive` and `name` options, ignore duplicate keywords, and reject empty lists or empty and whitespace-only keywords when the metric is created.
+
 `regex()` ignores the `g` and `y` flags, which would make identical outputs alternate between pass and fail.
 
 ### LLM-as-Judge
@@ -278,6 +298,7 @@ import { threshold } from '@cogitator-ai/evals';
 
 const assertions = [
   threshold('exactMatch', 0.9),
+  threshold('similarity', 0.8),
   threshold('latency', 5000),
   threshold('relevance', 0.7),
 ];
@@ -463,21 +484,24 @@ result.report('ci');
 
 ### Metrics
 
-| Export         | Type          | Description                         |
-| -------------- | ------------- | ----------------------------------- |
-| `exactMatch`   | Deterministic | Exact string match                  |
-| `contains`     | Deterministic | Substring match                     |
-| `regex`        | Deterministic | Regex pattern match                 |
-| `jsonSchema`   | Deterministic | Zod schema validation               |
-| `faithfulness` | LLM Judge     | Factual accuracy                    |
-| `relevance`    | LLM Judge     | Topical relevance                   |
-| `coherence`    | LLM Judge     | Logical structure                   |
-| `helpfulness`  | LLM Judge     | Practical usefulness                |
-| `llmMetric`    | LLM Judge     | Custom judge prompt                 |
-| `latency`      | Statistical   | Response time percentiles           |
-| `cost`         | Statistical   | Token cost aggregation              |
-| `tokenUsage`   | Statistical   | Input/output token counts           |
-| `metric`       | Custom        | Factory for domain-specific metrics |
+| Export         | Type          | Description                           |
+| -------------- | ------------- | ------------------------------------- |
+| `exactMatch`   | Deterministic | Exact string match                    |
+| `contains`     | Deterministic | Substring match                       |
+| `containsAll`  | Deterministic | Fraction of keywords found            |
+| `containsAny`  | Deterministic | Any configured keyword found          |
+| `similarity`   | Deterministic | Levenshtein distance similarity score |
+| `regex`        | Deterministic | Regex pattern match                   |
+| `jsonSchema`   | Deterministic | Zod schema validation                 |
+| `faithfulness` | LLM Judge     | Factual accuracy                      |
+| `relevance`    | LLM Judge     | Topical relevance                     |
+| `coherence`    | LLM Judge     | Logical structure                     |
+| `helpfulness`  | LLM Judge     | Practical usefulness                  |
+| `llmMetric`    | LLM Judge     | Custom judge prompt                   |
+| `latency`      | Statistical   | Response time percentiles             |
+| `cost`         | Statistical   | Token cost aggregation                |
+| `tokenUsage`   | Statistical   | Input/output token counts             |
+| `metric`       | Custom        | Factory for domain-specific metrics   |
 
 ### Assertions
 

package/dist/index.d.ts

@@ -7,7 +7,7 @@ export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
 export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
 export { latency, cost, tokenUsage } from './metrics/statistical.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -7,7 +7,7 @@ export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, regex, jsonSchema } from './metrics/deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
 export { latency, cost, tokenUsage } from './metrics/statistical.js';
 export { metric } from './metrics/custom.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/deterministic.d.ts

@@ -4,7 +4,12 @@ interface MatchOptions extends MetricOptions {
     caseSensitive?: boolean;
 }
 export declare function exactMatch(opts?: MatchOptions): MetricFn;
+export declare function similarity(opts?: MatchOptions): MetricFn;
 export declare function contains(opts?: MatchOptions): MetricFn;
+/** Scores the fraction of keywords found in the output. */
+export declare function containsAll(keywords: string[], opts?: MatchOptions): MetricFn;
+/** Scores 1 when the output contains at least one keyword, and 0 otherwise. */
+export declare function containsAny(keywords: string[], opts?: MatchOptions): MetricFn;
 /**
  * Scores 1 when the output matches `pattern`. The `g` and `y` flags are dropped: they make
  * `RegExp.test` resume from the previous match, so identical outputs would alternate between

package/dist/metrics/deterministic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAOD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAuCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAkCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkB7E;AAED,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAiB7E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}

package/dist/metrics/deterministic.js

@@ -2,6 +2,24 @@ function createMetricFn(name, fn) {
     fn.metricName = name;
     return fn;
 }
+function prepareKeywordTargets(metricName, keywords, caseSensitive) {
+    if (keywords.length === 0) {
+        throw new Error(`${metricName} requires at least one keyword`);
+    }
+    const targets = [];
+    const seen = new Set();
+    for (const keyword of keywords) {
+        if (keyword.trim().length === 0) {
+            throw new Error(`${metricName} requires non-empty keywords`);
+        }
+        const search = caseSensitive ? keyword : keyword.toLowerCase();
+        if (!seen.has(search)) {
+            seen.add(search);
+            targets.push({ original: keyword, search });
+        }
+    }
+    return targets;
+}
 export function exactMatch(opts) {
     const caseSensitive = opts?.caseSensitive ?? false;
     const name = opts?.name ?? 'exactMatch';
@@ -20,6 +38,61 @@ export function exactMatch(opts) {
         };
     }));
 }
+function levenshteinDistance(s1, s2) {
+    const m = s1.length;
+    const n = s2.length;
+    if (m === 0)
+        return n;
+    if (n === 0)
+        return m;
+    let prev = new Array(n + 1);
+    let curr = new Array(n + 1);
+    for (let j = 0; j <= n; j++) {
+        prev[j] = j;
+    }
+    for (let i = 1; i <= m; i++) {
+        curr[0] = i;
+        const char1 = s1[i - 1];
+        for (let j = 1; j <= n; j++) {
+            const cost = char1 === s2[j - 1] ? 0 : 1;
+            curr[j] = Math.min(prev[j] + 1, // deletion
+            curr[j - 1] + 1, // insertion
+            prev[j - 1] + cost // substitution
+            );
+        }
+        const temp = prev;
+        prev = curr;
+        curr = temp;
+    }
+    return prev[n];
+}
+export function similarity(opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'similarity';
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expected;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected value provided' };
+        }
+        const output = result.output.trim();
+        const target = expected.trim();
+        const s1 = caseSensitive ? output : output.toLowerCase();
+        const s2 = caseSensitive ? target : target.toLowerCase();
+        const outChars = [...s1];
+        const expChars = [...s2];
+        const maxLen = Math.max(outChars.length, expChars.length);
+        if (maxLen === 0) {
+            return { name, score: 1 };
+        }
+        const distance = levenshteinDistance(outChars, expChars);
+        const score = 1 - distance / maxLen;
+        return {
+            name,
+            score,
+            details: score < 1 ? `expected "${target}", got "${output}" (distance: ${distance})` : undefined,
+        };
+    }));
+}
 export function contains(opts) {
     const caseSensitive = opts?.caseSensitive ?? false;
     const name = opts?.name ?? 'contains';
@@ -38,6 +111,40 @@ export function contains(opts) {
         };
     }));
 }
+/** Scores the fraction of keywords found in the output. */
+export function containsAll(keywords, opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'containsAll';
+    const targets = prepareKeywordTargets('containsAll', keywords, caseSensitive);
+    return createMetricFn(name, (async (result) => {
+        const output = caseSensitive ? result.output : result.output.toLowerCase();
+        const missing = targets.filter((keyword) => !output.includes(keyword.search));
+        return {
+            name,
+            score: (targets.length - missing.length) / targets.length,
+            details: missing.length > 0
+                ? `missing keywords: ${missing.map((keyword) => JSON.stringify(keyword.original)).join(', ')}`
+                : undefined,
+        };
+    }));
+}
+/** Scores 1 when the output contains at least one keyword, and 0 otherwise. */
+export function containsAny(keywords, opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'containsAny';
+    const targets = prepareKeywordTargets('containsAny', keywords, caseSensitive);
+    return createMetricFn(name, (async (result) => {
+        const output = caseSensitive ? result.output : result.output.toLowerCase();
+        const found = targets.some((keyword) => output.includes(keyword.search));
+        return {
+            name,
+            score: found ? 1 : 0,
+            details: found
+                ? undefined
+                : `output does not contain any of: ${targets.map((keyword) => JSON.stringify(keyword.original)).join(', ')}`,
+        };
+    }));
+}
 /**
  * Scores 1 when the output matches `pattern`. The `g` and `y` flags are dropped: they make
  * `RegExp.test` resume from the previous match, so identical outputs would alternate between

package/dist/metrics/deterministic.js.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,qBAAqB,CAC5B,UAAyC,EACzC,QAAkB,EAClB,aAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gCAAgC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,WAAW;YACxB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY;YAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM;YACzD,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,qBAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC/G,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/index.d.ts

@@ -1,5 +1,5 @@
 export type { MetricScore, MetricUsage, MetricOptions, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, } from './types.js';
-export { exactMatch, contains, regex, jsonSchema } from './deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
 export type { CustomMetricConfig } from './custom.js';
 export { latency, cost, tokenUsage } from './statistical.js';

package/dist/metrics/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}

package/dist/metrics/index.js

@@ -1,4 +1,4 @@
-export { exactMatch, contains, regex, jsonSchema } from './deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
 export { latency, cost, tokenUsage } from './statistical.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';

package/dist/metrics/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/evals",
-  "version": "0.6.1",
+  "version": "0.8.0",
   "description": "Evaluation framework for Cogitator AI agents",
   "keywords": [
     "cogitator",
@@ -34,7 +34,7 @@
     "zod": "^4.6.5"
   },
   "peerDependencies": {
-    "@cogitator-ai/core": ">=0.39.1"
+    "@cogitator-ai/core": ">=0.53.0"
   },
   "peerDependenciesMeta": {
     "@cogitator-ai/core": {
@@ -45,7 +45,7 @@
     "papaparse": "^5.7.0"
   },
   "devDependencies": {
-    "@cogitator-ai/core": "0.39.1",
+    "@cogitator-ai/core": "0.53.0",
     "@types/papaparse": "^5.3.15",
     "typescript": "^6.0.3",
     "vitest": "^5.0.3"