@dawn-ai/evals 0.8.2 → 0.8.3

package/dist/index.d.ts

@@ -4,6 +4,6 @@ export { type LlmJudgeOptions, llmJudge } from "./llm-judge.js";
 export { resolveDataset } from "./resolve-dataset.js";
 export { type RunEvalOptions, runEval } from "./run-eval.js";
 export { type NormalizedScore, normalizeScore } from "./score.js";
-export { contains, custom, exactMatch, jsonEquals, regex, tokensUnder, toolCalled, } from "./scorers.js";
+export { contains, custom, exactMatch, jsonEquals, memoryFresh, memoryIsolated, memoryRecalled, regex, tokensUnder, toolCalled, } from "./scorers.js";
 export type { CaseResult, CaseScore, Dataset, EvalCase, EvalDefinition, EvalReport, GatePolicy, GateResult, Score, ScoredReport, Scorer, ScorerAggregate, } from "./types.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,KAAK,eAAe,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,cAAc,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EAAE,KAAK,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACjE,OAAO,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,UAAU,EACV,KAAK,EACL,WAAW,EACX,UAAU,GACX,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,cAAc,EACd,UAAU,EACV,UAAU,EACV,UAAU,EACV,KAAK,EACL,YAAY,EACZ,MAAM,EACN,eAAe,GAChB,MAAM,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,KAAK,eAAe,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,cAAc,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EAAE,KAAK,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACjE,OAAO,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,UAAU,EACV,WAAW,EACX,cAAc,EACd,cAAc,EACd,KAAK,EACL,WAAW,EACX,UAAU,GACX,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,cAAc,EACd,UAAU,EACV,UAAU,EACV,UAAU,EACV,KAAK,EACL,YAAY,EACZ,MAAM,EACN,eAAe,GAChB,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -4,4 +4,4 @@ export { llmJudge } from "./llm-judge.js";
 export { resolveDataset } from "./resolve-dataset.js";
 export { runEval } from "./run-eval.js";
 export { normalizeScore } from "./score.js";
-export { contains, custom, exactMatch, jsonEquals, regex, tokensUnder, toolCalled, } from "./scorers.js";
+export { contains, custom, exactMatch, jsonEquals, memoryFresh, memoryIsolated, memoryRecalled, regex, tokensUnder, toolCalled, } from "./scorers.js";

package/dist/scorers.d.ts

@@ -26,4 +26,25 @@ export declare function custom(fn: (run: AgentRunResult, testCase: EvalCase) =>
     name?: string;
     threshold?: number;
 }): Scorer;
+/**
+ * Score 1 if every id in `expectedIds` appears in at least one `recall` tool
+ * result string; score 0 with a reason listing missing ids.
+ */
+export declare function memoryRecalled(expectedIds: string[], opts?: {
+    threshold?: number;
+}): Scorer;
+/**
+ * Score 1 if `run.finalMessage` contains `expectedValue` (freshness check —
+ * the newer value surfaced in the response); else 0.
+ */
+export declare function memoryFresh(expectedValue: string, opts?: {
+    threshold?: number;
+}): Scorer;
+/**
+ * Score 1 if `forbidden` does NOT appear in any recall tool output or
+ * finalMessage (no cross-namespace leak); score 0 if it leaks.
+ */
+export declare function memoryIsolated(forbidden: string, opts?: {
+    threshold?: number;
+}): Scorer;
 //# sourceMappingURL=scorers.d.ts.map

package/dist/scorers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../src/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAMzD,uDAAuD;AACvD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMhE;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMvE;AAED,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAA;CAC1C,GAAG,MAAM,CAkBT;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE,MAAM,CAgBR;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,MAAM,CACpB,EAAE,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,EACvE,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,MAAM,CAMR"}
+{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../src/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAMzD,uDAAuD;AACvD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMhE;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMvE;AAED,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAA;CAC1C,GAAG,MAAM,CAkBT;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE,MAAM,CAgBR;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,MAAM,CACpB,EAAE,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,EACvE,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,MAAM,CAMR;AAaD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAY3F;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CASxF;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAcvF"}

package/dist/scorers.js

@@ -78,3 +78,62 @@ export function custom(fn, opts) {
         score: fn,
     };
 }
+// ---------------------------------------------------------------------------
+// Memory scorers
+// ---------------------------------------------------------------------------
+/** Returns the string content of all tool results whose name matches `toolName`. */
+function recallOutputs(run, toolName) {
+    return run.toolResults
+        .filter((r) => r.name === toolName)
+        .map((r) => (typeof r.content === "string" ? r.content : JSON.stringify(r.content)));
+}
+/**
+ * Score 1 if every id in `expectedIds` appears in at least one `recall` tool
+ * result string; score 0 with a reason listing missing ids.
+ */
+export function memoryRecalled(expectedIds, opts) {
+    return {
+        name: `memoryRecalled(${expectedIds.join(",")})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => {
+            const outputs = recallOutputs(run, "recall");
+            const combined = outputs.join("\n");
+            const missing = expectedIds.filter((id) => !combined.includes(id));
+            if (missing.length === 0)
+                return 1;
+            return { score: 0, reason: `Missing memory ids: ${missing.join(", ")}` };
+        },
+    };
+}
+/**
+ * Score 1 if `run.finalMessage` contains `expectedValue` (freshness check —
+ * the newer value surfaced in the response); else 0.
+ */
+export function memoryFresh(expectedValue, opts) {
+    return {
+        name: `memoryFresh(${expectedValue})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => run.finalMessage.includes(expectedValue)
+            ? 1
+            : { score: 0, reason: `Expected "${expectedValue}" in finalMessage` },
+    };
+}
+/**
+ * Score 1 if `forbidden` does NOT appear in any recall tool output or
+ * finalMessage (no cross-namespace leak); score 0 if it leaks.
+ */
+export function memoryIsolated(forbidden, opts) {
+    return {
+        name: `memoryIsolated(${forbidden})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => {
+            const outputs = recallOutputs(run, "recall");
+            const leaksInRecall = outputs.some((o) => o.includes(forbidden));
+            const leaksInMessage = run.finalMessage.includes(forbidden);
+            if (leaksInRecall || leaksInMessage) {
+                return { score: 0, reason: `Cross-namespace leak: "${forbidden}" found in output` };
+            }
+            return 1;
+        },
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dawn-ai/evals",
-  "version": "0.8.2",
+  "version": "0.8.3",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -30,12 +30,12 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@dawn-ai/testing": "0.8.2"
+    "@dawn-ai/testing": "0.8.3"
   },
   "devDependencies": {
     "@types/node": "25.6.0",
-    "@dawn-ai/testing": "0.8.2",
-    "@dawn-ai/config-typescript": "0.8.2"
+    "@dawn-ai/testing": "0.8.3",
+    "@dawn-ai/config-typescript": "0.8.3"
   },
   "scripts": {
     "build": "tsc -b tsconfig.json",