@agentuity/evals 0.0.103

package/dist/answer-completeness.d.ts

@@ -0,0 +1,24 @@
+import type { BaseEvalOptions } from './types';
+export declare const answerCompletenessPrompt = "You are evaluating whether an LLM response directly addresses the user's request.\n\n## Inputs\n\n- USER REQUEST: {{USER_REQUEST}}\n- MODEL RESPONSE: {{MODEL_RESPONSE}}\n\n## Your task\n\n1. Assume a strict auditor who expects every part of the user's request to be addressed.\n2. Identify each distinct question, instruction, or requirement in the USER REQUEST.\n3. For each identified item, check whether the MODEL RESPONSE provides a direct, relevant answer or fulfillment.\n4. Flag any of the following violations:\n   - Ignoring part of the request entirely\n   - Providing tangential information instead of answering\n   - Deflecting with \"I can't help with that\" without justification\n   - Answering a different question than what was asked\n   - Providing vague or generic responses that do not address specifics\n   - Excessive hedging or caveats that obscure whether the question was answered\n\n## Score\n\n- Start from 1.0.\n- Subtract points for each violation:\n   - Minor omission (small detail or sub-question not addressed): \u22120.2\n   - Partial answer (core question addressed but incompletely): \u22120.3\n   - Tangential response (related content but does not answer the question): \u22120.5\n   - Complete miss (major requirement or question ignored entirely): \u22120.6\n   - Refusal without valid justification: \u22120.8\n- Minimum score is 0.0.\n- Multiple violations compound independently.\n\n## Pass/Fail\n\n- passed = true only if score \u2265 0.7 AND no complete misses or unjustified refusals are present.\n\n## Constraints\n\n- Do not credit the response for being correct if it does not address what was asked.\n- Do not credit the response for being helpful on unrelated topics.\n- Do not infer that the user's needs were met unless explicitly addressed in the response.\n- Do not excuse incomplete answers due to response length or complexity.\n\n## Output format (STRICT JSON, one line reason):\n\n{\n  \"score\": <number between 0.0 and 1.0>,\n  \"passed\": <true|false>,\n  \"metadata\": {\n    \"reason\": \"<single concise sentence listing which parts of the request were or were not addressed>\"\n  }\n}";
+type AnswerCompletenessEvalOptions = BaseEvalOptions & {
+    threshold: number;
+};
+export declare const answerCompleteness: <TAgentInput extends import("@agentuity/core").StandardSchemaV1 | undefined = any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1 | undefined = any>(overrides?: (Partial<{
+    name?: string;
+    description?: string;
+} & BaseEvalOptions & {
+    threshold: number;
+}> & {
+    middleware?: import("./types").EvalMiddleware<TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, {
+        request: string;
+    } & {
+        context?: string | undefined;
+    }, {
+        response: string;
+    } & {}> | undefined;
+}) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
+    name: string;
+    options: AnswerCompletenessEvalOptions;
+};
+export {};
+//# sourceMappingURL=answer-completeness.d.ts.map

package/dist/answer-completeness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer-completeness.d.ts","sourceRoot":"","sources":["../src/answer-completeness.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,eAAO,MAAM,wBAAwB,2mEAmDnC,CAAC;AAEH,KAAK,6BAA6B,GAAG,eAAe,GAAG;IACtD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;eAHnB,MAAM;;;;;;;;;;;;CAgChB,CAAC"}

package/dist/answer-completeness.js

@@ -0,0 +1,79 @@
+import { openai } from '@ai-sdk/openai';
+import { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
+export const answerCompletenessPrompt = `You are evaluating whether an LLM response directly addresses the user's request.
+
+## Inputs
+
+- USER REQUEST: {{USER_REQUEST}}
+- MODEL RESPONSE: {{MODEL_RESPONSE}}
+
+## Your task
+
+1. Assume a strict auditor who expects every part of the user's request to be addressed.
+2. Identify each distinct question, instruction, or requirement in the USER REQUEST.
+3. For each identified item, check whether the MODEL RESPONSE provides a direct, relevant answer or fulfillment.
+4. Flag any of the following violations:
+   - Ignoring part of the request entirely
+   - Providing tangential information instead of answering
+   - Deflecting with "I can't help with that" without justification
+   - Answering a different question than what was asked
+   - Providing vague or generic responses that do not address specifics
+   - Excessive hedging or caveats that obscure whether the question was answered
+
+## Score
+
+- Start from 1.0.
+- Subtract points for each violation:
+   - Minor omission (small detail or sub-question not addressed): −0.2
+   - Partial answer (core question addressed but incompletely): −0.3
+   - Tangential response (related content but does not answer the question): −0.5
+   - Complete miss (major requirement or question ignored entirely): −0.6
+   - Refusal without valid justification: −0.8
+- Minimum score is 0.0.
+- Multiple violations compound independently.
+
+## Pass/Fail
+
+- passed = true only if score ≥ 0.7 AND no complete misses or unjustified refusals are present.
+
+## Constraints
+
+- Do not credit the response for being correct if it does not address what was asked.
+- Do not credit the response for being helpful on unrelated topics.
+- Do not infer that the user's needs were met unless explicitly addressed in the response.
+- Do not excuse incomplete answers due to response length or complexity.
+
+## Output format (STRICT JSON, one line reason):
+
+{
+  "score": <number between 0.0 and 1.0>,
+  "passed": <true|false>,
+  "metadata": {
+    "reason": "<single concise sentence listing which parts of the request were or were not addressed>"
+  }
+}`;
+export const answerCompleteness = createPresetEval({
+    name: 'answer-completeness',
+    description: 'Evaluates whether response fully addresses all parts of the user request',
+    options: {
+        model: openai('gpt-4o'),
+        threshold: 0.7,
+    },
+    handler: async (ctx, input, output, options) => {
+        const prompt = interpolatePrompt(answerCompletenessPrompt, {
+            USER_REQUEST: input.request,
+            MODEL_RESPONSE: output.response,
+        });
+        const evaluation = await generateEvalResult({ model: options.model, prompt });
+        return {
+            passed: evaluation.passed && (evaluation.score ?? 1) >= options.threshold,
+            score: evaluation.score,
+            metadata: {
+                ...evaluation.metadata,
+                model: options.model,
+                threshold: options.threshold,
+            },
+        };
+    },
+});
+//# sourceMappingURL=answer-completeness.js.map

package/dist/answer-completeness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer-completeness.js","sourceRoot":"","sources":["../src/answer-completeness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAGlB,MAAM,UAAU,CAAC;AAGlB,MAAM,CAAC,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmDtC,CAAC;AAMH,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAIhD;IACD,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EAAE,0EAA0E;IACvF,OAAO,EAAE;QACR,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;QACvB,SAAS,EAAE,GAAG;KACd;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,wBAAwB,EAAE;YAC1D,YAAY,EAAE,KAAK,CAAC,OAAO;YAC3B,cAAc,EAAE,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9E,OAAO;YACN,MAAM,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS;YACzE,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE;gBACT,GAAG,UAAU,CAAC,QAAQ;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;aAC5B;SACD,CAAC;IACH,CAAC;CACD,CAAC,CAAC"}

package/dist/conciseness.d.ts

@@ -0,0 +1,24 @@
+import type { BaseEvalOptions } from './types';
+export declare const concisenessPrompt = "You are evaluating whether an LLM response is unnecessarily verbose for the request type.\n\n## Inputs\n\n- USER REQUEST: {{USER_REQUEST}}\n- MODEL RESPONSE: {{MODEL_RESPONSE}}\n\n## Your task\n\n1. Assume a reader who values efficiency and dislikes wasted words.\n2. Assess the complexity and scope of the USER REQUEST to determine appropriate response length.\n3. Identify any of the following verbosity violations in the MODEL RESPONSE:\n   - Repeating the same information in different words\n   - Excessive preamble before addressing the request\n   - Unnecessary summaries or recaps at the end\n   - Filler phrases that add no meaning (e.g., \"It's important to note that...\", \"As you may know...\")\n   - Over-explanation of simple concepts the user likely understands\n   - Including tangential context not required to answer the question\n   - Excessive hedging or qualifiers beyond what accuracy requires\n\n## Score\n\n- Start from 1.0.\n- Subtract points for each violation:\n   - Minor filler or unnecessary phrase: \u22120.1\n   - Redundant paragraph or repeated explanation: \u22120.3\n   - Excessive preamble or postamble (multiple sentences): \u22120.3\n   - Response length grossly disproportionate to request simplicity: \u22120.5\n- Minimum score is 0.0.\n- Multiple violations compound independently.\n\n## Pass/Fail\n\n- passed = true only if score \u2265 0.7 AND response length is reasonably proportionate to request complexity.\n\n## Constraints\n\n- Do not penalize thoroughness when the request genuinely requires detailed explanation.\n- Do not credit brevity if it sacrifices completeness.\n- Do not assume the user wants verbose responses unless explicitly requested.\n- Judge verbosity relative to what a competent, efficient response would require.\n\n## Output format (STRICT JSON, one line reason):\n\n{\n  \"score\": <number between 0.0 and 1.0>,\n  \"passed\": <true|false>,\n  \"metadata\": {\n    \"reason\": \"<single concise sentence describing verbosity issues found or confirming appropriate length>\"\n  }\n}";
+type ConcisenessEvalOptions = BaseEvalOptions & {
+    threshold: number;
+};
+export declare const conciseness: <TAgentInput extends import("@agentuity/core").StandardSchemaV1 | undefined = any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1 | undefined = any>(overrides?: (Partial<{
+    name?: string;
+    description?: string;
+} & BaseEvalOptions & {
+    threshold: number;
+}> & {
+    middleware?: import("./types").EvalMiddleware<TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, {
+        request: string;
+    } & {
+        context?: string | undefined;
+    }, {
+        response: string;
+    } & {}> | undefined;
+}) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
+    name: string;
+    options: ConcisenessEvalOptions;
+};
+export {};
+//# sourceMappingURL=conciseness.d.ts.map

package/dist/conciseness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conciseness.d.ts","sourceRoot":"","sources":["../src/conciseness.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,eAAO,MAAM,iBAAiB,0gEAkD5B,CAAC;AAEH,KAAK,sBAAsB,GAAG,eAAe,GAAG;IAC/C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,WAAW;;;;eAHZ,MAAM;;;;;;;;;;;;CAgChB,CAAC"}

package/dist/conciseness.js

@@ -0,0 +1,78 @@
+import { openai } from '@ai-sdk/openai';
+import { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
+export const concisenessPrompt = `You are evaluating whether an LLM response is unnecessarily verbose for the request type.
+
+## Inputs
+
+- USER REQUEST: {{USER_REQUEST}}
+- MODEL RESPONSE: {{MODEL_RESPONSE}}
+
+## Your task
+
+1. Assume a reader who values efficiency and dislikes wasted words.
+2. Assess the complexity and scope of the USER REQUEST to determine appropriate response length.
+3. Identify any of the following verbosity violations in the MODEL RESPONSE:
+   - Repeating the same information in different words
+   - Excessive preamble before addressing the request
+   - Unnecessary summaries or recaps at the end
+   - Filler phrases that add no meaning (e.g., "It's important to note that...", "As you may know...")
+   - Over-explanation of simple concepts the user likely understands
+   - Including tangential context not required to answer the question
+   - Excessive hedging or qualifiers beyond what accuracy requires
+
+## Score
+
+- Start from 1.0.
+- Subtract points for each violation:
+   - Minor filler or unnecessary phrase: −0.1
+   - Redundant paragraph or repeated explanation: −0.3
+   - Excessive preamble or postamble (multiple sentences): −0.3
+   - Response length grossly disproportionate to request simplicity: −0.5
+- Minimum score is 0.0.
+- Multiple violations compound independently.
+
+## Pass/Fail
+
+- passed = true only if score ≥ 0.7 AND response length is reasonably proportionate to request complexity.
+
+## Constraints
+
+- Do not penalize thoroughness when the request genuinely requires detailed explanation.
+- Do not credit brevity if it sacrifices completeness.
+- Do not assume the user wants verbose responses unless explicitly requested.
+- Judge verbosity relative to what a competent, efficient response would require.
+
+## Output format (STRICT JSON, one line reason):
+
+{
+  "score": <number between 0.0 and 1.0>,
+  "passed": <true|false>,
+  "metadata": {
+    "reason": "<single concise sentence describing verbosity issues found or confirming appropriate length>"
+  }
+}`;
+export const conciseness = createPresetEval({
+    name: 'conciseness',
+    description: 'Evaluates whether response is appropriately concise without unnecessary verbosity',
+    options: {
+        model: openai('gpt-4o'),
+        threshold: 0.7,
+    },
+    handler: async (ctx, input, output, options) => {
+        const prompt = interpolatePrompt(concisenessPrompt, {
+            USER_REQUEST: input.request,
+            MODEL_RESPONSE: output.response,
+        });
+        const evaluation = await generateEvalResult({ model: options.model, prompt });
+        return {
+            passed: evaluation.passed && (evaluation.score ?? 1) >= options.threshold,
+            score: evaluation.score,
+            metadata: {
+                ...evaluation.metadata,
+                model: options.model,
+                threshold: options.threshold,
+            },
+        };
+    },
+});
+//# sourceMappingURL=conciseness.js.map

package/dist/conciseness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"conciseness.js","sourceRoot":"","sources":["../src/conciseness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAGlB,MAAM,UAAU,CAAC;AAGlB,MAAM,CAAC,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkD/B,CAAC;AAMH,MAAM,CAAC,MAAM,WAAW,GAAG,gBAAgB,CAIzC;IACD,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,mFAAmF;IAChG,OAAO,EAAE;QACR,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;QACvB,SAAS,EAAE,GAAG;KACd;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,iBAAiB,EAAE;YACnD,YAAY,EAAE,KAAK,CAAC,OAAO;YAC3B,cAAc,EAAE,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9E,OAAO;YACN,MAAM,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS;YACzE,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE;gBACT,GAAG,UAAU,CAAC,QAAQ;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;aAC5B;SACD,CAAC;IACH,CAAC;CACD,CAAC,CAAC"}

package/dist/extraneous-content.d.ts

@@ -0,0 +1,24 @@
+import type { BaseEvalOptions } from './types';
+export declare const extraneousContentPrompt = "You are evaluating whether an LLM response contains content clearly unrelated or unnecessary for fulfilling the request.\n\n## Inputs\n\n- USER REQUEST: {{USER_REQUEST}}\n- MODEL RESPONSE: {{MODEL_RESPONSE}}\n\n## Your task\n\n1. Assume a strict editor who removes anything not directly serving the user's goal.\n2. Identify each distinct section, paragraph, or statement in the MODEL RESPONSE.\n3. For each element, determine whether it contributes to answering the USER REQUEST.\n4. Flag any of the following as extraneous content:\n   - Topics or information not requested and not necessary for context\n   - Unsolicited advice or recommendations beyond the scope of the request\n   - Self-promotional statements about capabilities\n   - Unnecessary caveats or warnings unrelated to the specific request\n   - Meta-commentary about the response itself (e.g., \"I hope this helps!\")\n   - Offers to help with additional unrelated tasks\n   - Background information the user clearly already knows based on their request\n   - Repeated information already stated elsewhere in the response\n\n## Score\n\n- Start from 1.0.\n- Subtract points for each extraneous element:\n   - Brief unnecessary phrase or sentence: \u22120.2\n   - Full paragraph of off-topic content: \u22120.4\n   - Multiple paragraphs or significant tangent: \u22120.6\n- Minimum score is 0.0.\n- Multiple violations compound independently.\n\n## Pass/Fail\n\n- passed = true only if score \u2265 {{THRESHOLD}} AND no major tangents or significant off-topic content is present.\n\n## Constraints\n\n- Do not credit relevant information for excusing unrelated additions.\n- Do not assume the user wants additional context unless requested.\n- Do not excuse extraneous content because it might be \"useful\" to some readers.\n- Helpful-but-unsolicited content is still extraneous if not requested.\n- Necessary context to understand the answer is not extraneous.\n\n## Output format (STRICT JSON, one line reason):\n\n{\n  \"score\": <number between 0.0 and 1.0>,\n  \"passed\": <true|false>,\n  \"metadata\": {\n    \"reason\": \"<single concise sentence listing extraneous content found or confirming all content was relevant>\"\n  }\n}";
+type ExtraneousContentEvalOptions = BaseEvalOptions & {
+    threshold: number;
+};
+export declare const extraneousContent: <TAgentInput extends import("@agentuity/core").StandardSchemaV1 | undefined = any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1 | undefined = any>(overrides?: (Partial<{
+    name?: string;
+    description?: string;
+} & BaseEvalOptions & {
+    threshold: number;
+}> & {
+    middleware?: import("./types").EvalMiddleware<TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, {
+        request: string;
+    } & {
+        context?: string | undefined;
+    }, {
+        response: string;
+    } & {}> | undefined;
+}) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
+    name: string;
+    options: ExtraneousContentEvalOptions;
+};
+export {};
+//# sourceMappingURL=extraneous-content.d.ts.map

package/dist/extraneous-content.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraneous-content.d.ts","sourceRoot":"","sources":["../src/extraneous-content.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,eAAO,MAAM,uBAAuB,kqEAoDlC,CAAC;AAEH,KAAK,4BAA4B,GAAG,eAAe,GAAG;IACrD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;eAHlB,MAAM;;;;;;;;;;;;CAiChB,CAAC"}

package/dist/extraneous-content.js

@@ -0,0 +1,81 @@
+import { openai } from '@ai-sdk/openai';
+import { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
+export const extraneousContentPrompt = `You are evaluating whether an LLM response contains content clearly unrelated or unnecessary for fulfilling the request.
+
+## Inputs
+
+- USER REQUEST: {{USER_REQUEST}}
+- MODEL RESPONSE: {{MODEL_RESPONSE}}
+
+## Your task
+
+1. Assume a strict editor who removes anything not directly serving the user's goal.
+2. Identify each distinct section, paragraph, or statement in the MODEL RESPONSE.
+3. For each element, determine whether it contributes to answering the USER REQUEST.
+4. Flag any of the following as extraneous content:
+   - Topics or information not requested and not necessary for context
+   - Unsolicited advice or recommendations beyond the scope of the request
+   - Self-promotional statements about capabilities
+   - Unnecessary caveats or warnings unrelated to the specific request
+   - Meta-commentary about the response itself (e.g., "I hope this helps!")
+   - Offers to help with additional unrelated tasks
+   - Background information the user clearly already knows based on their request
+   - Repeated information already stated elsewhere in the response
+
+## Score
+
+- Start from 1.0.
+- Subtract points for each extraneous element:
+   - Brief unnecessary phrase or sentence: −0.2
+   - Full paragraph of off-topic content: −0.4
+   - Multiple paragraphs or significant tangent: −0.6
+- Minimum score is 0.0.
+- Multiple violations compound independently.
+
+## Pass/Fail
+
+- passed = true only if score ≥ {{THRESHOLD}} AND no major tangents or significant off-topic content is present.
+
+## Constraints
+
+- Do not credit relevant information for excusing unrelated additions.
+- Do not assume the user wants additional context unless requested.
+- Do not excuse extraneous content because it might be "useful" to some readers.
+- Helpful-but-unsolicited content is still extraneous if not requested.
+- Necessary context to understand the answer is not extraneous.
+
+## Output format (STRICT JSON, one line reason):
+
+{
+  "score": <number between 0.0 and 1.0>,
+  "passed": <true|false>,
+  "metadata": {
+    "reason": "<single concise sentence listing extraneous content found or confirming all content was relevant>"
+  }
+}`;
+export const extraneousContent = createPresetEval({
+    name: 'extraneous-content',
+    description: 'Evaluates whether response contains unnecessary or off-topic content',
+    options: {
+        model: openai('gpt-4o'),
+        threshold: 0.7,
+    },
+    handler: async (ctx, input, output, options) => {
+        const prompt = interpolatePrompt(extraneousContentPrompt, {
+            USER_REQUEST: input.request,
+            MODEL_RESPONSE: output.response,
+            THRESHOLD: options.threshold.toString(),
+        });
+        const evaluation = await generateEvalResult({ model: options.model, prompt });
+        return {
+            passed: evaluation.passed && (evaluation.score ?? 1) >= options.threshold,
+            score: evaluation.score,
+            metadata: {
+                ...evaluation.metadata,
+                model: options.model,
+                threshold: options.threshold,
+            },
+        };
+    },
+});
+//# sourceMappingURL=extraneous-content.js.map

package/dist/extraneous-content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraneous-content.js","sourceRoot":"","sources":["../src/extraneous-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAGlB,MAAM,UAAU,CAAC;AAGlB,MAAM,CAAC,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoDrC,CAAC;AAMH,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAI/C;IACD,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,sEAAsE;IACnF,OAAO,EAAE;QACR,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;QACvB,SAAS,EAAE,GAAG;KACd;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,uBAAuB,EAAE;YACzD,YAAY,EAAE,KAAK,CAAC,OAAO;YAC3B,cAAc,EAAE,MAAM,CAAC,QAAQ;YAC/B,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9E,OAAO;YACN,MAAM,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS;YACzE,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE;gBACT,GAAG,UAAU,CAAC,QAAQ;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;aAC5B;SACD,CAAC;IACH,CAAC;CACD,CAAC,CAAC"}

package/dist/format.d.ts

@@ -0,0 +1,18 @@
+import type { BaseEvalOptions } from './types';
+export declare const formatPrompt = "You are evaluating whether an LLM response matches the format requested by the user.\n\n## Inputs\n\n- USER REQUEST: {{USER_REQUEST}}\n- MODEL RESPONSE: {{MODEL_RESPONSE}}\n\n## Your task\n\n1. Assume a strict validator checking format compliance.\n2. Identify any explicit format requirements in the USER REQUEST, including:\n   - Structured data formats: JSON, XML, YAML, CSV, etc.\n   - Document formats: Markdown, plain text, HTML, etc.\n   - List formats: Bullet points, numbered lists, comma-separated, etc.\n   - Table formats: Markdown tables, ASCII tables, etc.\n   - Code formats: Specific programming language, code blocks, etc.\n   - Length constraints: Word counts, character limits, number of items, etc.\n   - Structural requirements: Sections, headers, specific fields, etc.\n3. If no format is explicitly requested, this eval automatically passes.\n4. If a format is requested, verify the MODEL RESPONSE strictly adheres to it:\n   - JSON must be valid, parseable JSON\n   - Lists must use the specified list style\n   - Tables must have proper structure\n   - Code must be in the specified language and properly formatted\n   - Length constraints must be met exactly or within stated tolerance\n\n## Pass/Fail\n\n- passed = true only if no format was requested OR the response strictly matches all requested format requirements.\n- passed = false if any format requirement is violated, even partially.\n\n## Constraints\n\n- Do not assume implicit format preferences; only enforce explicit requests.\n- Do not credit \"close enough\" formatting; requirements must be met exactly.\n- Do not excuse format violations because the content is otherwise correct.\n- Do not pass responses that wrap requested format in additional commentary unless explicitly allowed.\n- JSON responses with syntax errors (trailing commas, unquoted keys, etc.) are failures.\n\n## Output format (STRICT JSON, one line reason):\n\n{\n  \"passed\": <true|false>,\n  \"metadata\": {\n    \"reason\": \"<single concise sentence stating format requirement and whether it was met, or confirming no format was requested>\"\n  }\n}";
+export declare const format: <TAgentInput extends import("@agentuity/core").StandardSchemaV1 | undefined = any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1 | undefined = any>(overrides?: (Partial<{
+    name?: string;
+    description?: string;
+} & BaseEvalOptions> & {
+    middleware?: import("./types").EvalMiddleware<TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, {
+        request: string;
+    } & {
+        context?: string | undefined;
+    }, {
+        response: string;
+    } & {}> | undefined;
+}) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
+    name: string;
+    options: BaseEvalOptions;
+};
+//# sourceMappingURL=format.d.ts.map

package/dist/format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,eAAO,MAAM,YAAY,0kEA8CvB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;CAsBjB,CAAC"}

package/dist/format.js

@@ -0,0 +1,71 @@
+import { openai } from '@ai-sdk/openai';
+import { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
+export const formatPrompt = `You are evaluating whether an LLM response matches the format requested by the user.
+
+## Inputs
+
+- USER REQUEST: {{USER_REQUEST}}
+- MODEL RESPONSE: {{MODEL_RESPONSE}}
+
+## Your task
+
+1. Assume a strict validator checking format compliance.
+2. Identify any explicit format requirements in the USER REQUEST, including:
+   - Structured data formats: JSON, XML, YAML, CSV, etc.
+   - Document formats: Markdown, plain text, HTML, etc.
+   - List formats: Bullet points, numbered lists, comma-separated, etc.
+   - Table formats: Markdown tables, ASCII tables, etc.
+   - Code formats: Specific programming language, code blocks, etc.
+   - Length constraints: Word counts, character limits, number of items, etc.
+   - Structural requirements: Sections, headers, specific fields, etc.
+3. If no format is explicitly requested, this eval automatically passes.
+4. If a format is requested, verify the MODEL RESPONSE strictly adheres to it:
+   - JSON must be valid, parseable JSON
+   - Lists must use the specified list style
+   - Tables must have proper structure
+   - Code must be in the specified language and properly formatted
+   - Length constraints must be met exactly or within stated tolerance
+
+## Pass/Fail
+
+- passed = true only if no format was requested OR the response strictly matches all requested format requirements.
+- passed = false if any format requirement is violated, even partially.
+
+## Constraints
+
+- Do not assume implicit format preferences; only enforce explicit requests.
+- Do not credit "close enough" formatting; requirements must be met exactly.
+- Do not excuse format violations because the content is otherwise correct.
+- Do not pass responses that wrap requested format in additional commentary unless explicitly allowed.
+- JSON responses with syntax errors (trailing commas, unquoted keys, etc.) are failures.
+
+## Output format (STRICT JSON, one line reason):
+
+{
+  "passed": <true|false>,
+  "metadata": {
+    "reason": "<single concise sentence stating format requirement and whether it was met, or confirming no format was requested>"
+  }
+}`;
+export const format = createPresetEval({
+    name: 'format',
+    description: 'Evaluates whether response matches the requested format',
+    options: {
+        model: openai('gpt-4o'),
+    },
+    handler: async (ctx, input, output, options) => {
+        const prompt = interpolatePrompt(formatPrompt, {
+            USER_REQUEST: input.request,
+            MODEL_RESPONSE: output.response,
+        });
+        const evaluation = await generateEvalResult({ model: options.model, prompt });
+        return {
+            passed: evaluation.passed,
+            metadata: {
+                ...evaluation.metadata,
+                model: options.model,
+            },
+        };
+    },
+});
+//# sourceMappingURL=format.js.map

package/dist/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAGlB,MAAM,UAAU,CAAC;AAGlB,MAAM,CAAC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA8C1B,CAAC;AAEH,MAAM,CAAC,MAAM,MAAM,GAAG,gBAAgB,CAAuD;IAC5F,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,yDAAyD;IACtE,OAAO,EAAE;QACR,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;KACvB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,EAAE;YAC9C,YAAY,EAAE,KAAK,CAAC,OAAO;YAC3B,cAAc,EAAE,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9E,OAAO;YACN,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE;gBACT,GAAG,UAAU,CAAC,QAAQ;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;aACpB;SACD,CAAC;IACH,CAAC;CACD,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+export { createPresetEval, interpolatePrompt, generateEvalResult, type DefaultEvalInput, type DefaultEvalOutput, type GenerateEvalResultOptions, } from './_utils';
+export type { BaseEvalOptions, EvalMiddleware } from './types';
+export { politeness, politenessPrompt } from './politeness';
+export { safety, safetyPrompt } from './safety';
+export { pii, piiPrompt } from './pii';
+export { conciseness, concisenessPrompt } from './conciseness';
+export { adversarial, adversarialPrompt } from './adversarial';
+export { ambiguity, ambiguityPrompt } from './ambiguity';
+export { answerCompleteness, answerCompletenessPrompt } from './answer-completeness';
+export { extraneousContent, extraneousContentPrompt } from './extraneous-content';
+export { format, formatPrompt } from './format';
+export { knowledgeRetention, knowledgeRetentionPrompt } from './knowledge-retention';
+export { roleAdherence, roleAdherencePrompt } from './role-adherence';
+export { selfReference, selfReferencePrompt } from './self-reference';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC9B,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAG/D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/index.js

@@ -0,0 +1,15 @@
+export { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
+// Evals (each file contains both the prompt and the eval)
+export { politeness, politenessPrompt } from './politeness';
+export { safety, safetyPrompt } from './safety';
+export { pii, piiPrompt } from './pii';
+export { conciseness, concisenessPrompt } from './conciseness';
+export { adversarial, adversarialPrompt } from './adversarial';
+export { ambiguity, ambiguityPrompt } from './ambiguity';
+export { answerCompleteness, answerCompletenessPrompt } from './answer-completeness';
+export { extraneousContent, extraneousContentPrompt } from './extraneous-content';
+export { format, formatPrompt } from './format';
+export { knowledgeRetention, knowledgeRetentionPrompt } from './knowledge-retention';
+export { roleAdherence, roleAdherencePrompt } from './role-adherence';
+export { selfReference, selfReferencePrompt } from './self-reference';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAIlB,MAAM,UAAU,CAAC;AAGlB,0DAA0D;AAC1D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/knowledge-retention.d.ts

@@ -0,0 +1,24 @@
+import type { BaseEvalOptions } from './types';
+export declare const knowledgeRetentionPrompt = "You are evaluating whether an LLM response correctly retains and applies facts or decisions provided earlier in the conversation.\n\n## Inputs\n\n- USER REQUEST (may include conversation history or prior context): {{USER_REQUEST}}\n- MODEL RESPONSE: {{MODEL_RESPONSE}}\n\n## Your task\n\n1. Assume an auditor checking for context consistency and memory accuracy.\n2. Identify all facts, decisions, preferences, or constraints established in the USER REQUEST or prior context, including:\n   - Names, dates, numbers, or specific values mentioned\n   - User preferences or requirements stated earlier\n   - Decisions or conclusions reached in prior exchanges\n   - Constraints or boundaries defined for the task\n   - Corrections or clarifications the user provided\n3. Check the MODEL RESPONSE for any of the following retention failures:\n   - Contradicting previously established facts\n   - Ignoring stated preferences or requirements\n   - Using incorrect values for previously defined variables\n   - Forgetting constraints that should limit the response\n   - Asking for information already provided\n   - Reverting to defaults when specific choices were made\n\n## Score\n\n- Start from 1.0.\n- Subtract points for each retention failure:\n   - Minor detail forgotten (peripheral to main task): \u22120.2\n   - Preference or requirement ignored: \u22120.4\n   - Key fact contradicted or misremembered: \u22120.5\n   - Critical constraint violated: \u22120.6\n- Minimum score is 0.0.\n- Multiple failures compound independently.\n\n## Pass/Fail\n\n- passed = true only if score \u2265 0.7 AND no critical facts are contradicted or key constraints violated.\n\n## Constraints\n\n- Do not assume the response \"probably meant\" the correct information.\n- Do not excuse retention failures because the response is otherwise helpful.\n- Do not credit partial retention if critical elements are missed.\n- If no prior context is provided, this eval automatically passes with score 1.0.\n- Evaluate only retention of information explicitly stated, not implied.\n\n## Output format (STRICT JSON, one line reason):\n\n{\n  \"score\": <number between 0.0 and 1.0>,\n  \"passed\": <true|false>,\n  \"metadata\": {\n    \"reason\": \"<single concise sentence listing retention failures found or confirming context was correctly maintained>\"\n  }\n}";
+type KnowledgeRetentionEvalOptions = BaseEvalOptions & {
+    threshold: number;
+};
+export declare const knowledgeRetention: <TAgentInput extends import("@agentuity/core").StandardSchemaV1 | undefined = any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1 | undefined = any>(overrides?: (Partial<{
+    name?: string;
+    description?: string;
+} & BaseEvalOptions & {
+    threshold: number;
+}> & {
+    middleware?: import("./types").EvalMiddleware<TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, {
+        request: string;
+    } & {
+        context?: string | undefined;
+    }, {
+        response: string;
+    } & {}> | undefined;
+}) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
+    name: string;
+    options: KnowledgeRetentionEvalOptions;
+};
+export {};
+//# sourceMappingURL=knowledge-retention.d.ts.map

package/dist/knowledge-retention.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledge-retention.d.ts","sourceRoot":"","sources":["../src/knowledge-retention.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,eAAO,MAAM,wBAAwB,4yEAuDnC,CAAC;AAEH,KAAK,6BAA6B,GAAG,eAAe,GAAG;IACtD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;eAHnB,MAAM;;;;;;;;;;;;CAgChB,CAAC"}

package/dist/knowledge-retention.js

@@ -0,0 +1,83 @@
+import { openai } from '@ai-sdk/openai';
+import { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
+export const knowledgeRetentionPrompt = `You are evaluating whether an LLM response correctly retains and applies facts or decisions provided earlier in the conversation.
+
+## Inputs
+
+- USER REQUEST (may include conversation history or prior context): {{USER_REQUEST}}
+- MODEL RESPONSE: {{MODEL_RESPONSE}}
+
+## Your task
+
+1. Assume an auditor checking for context consistency and memory accuracy.
+2. Identify all facts, decisions, preferences, or constraints established in the USER REQUEST or prior context, including:
+   - Names, dates, numbers, or specific values mentioned
+   - User preferences or requirements stated earlier
+   - Decisions or conclusions reached in prior exchanges
+   - Constraints or boundaries defined for the task
+   - Corrections or clarifications the user provided
+3. Check the MODEL RESPONSE for any of the following retention failures:
+   - Contradicting previously established facts
+   - Ignoring stated preferences or requirements
+   - Using incorrect values for previously defined variables
+   - Forgetting constraints that should limit the response
+   - Asking for information already provided
+   - Reverting to defaults when specific choices were made
+
+## Score
+
+- Start from 1.0.
+- Subtract points for each retention failure:
+   - Minor detail forgotten (peripheral to main task): −0.2
+   - Preference or requirement ignored: −0.4
+   - Key fact contradicted or misremembered: −0.5
+   - Critical constraint violated: −0.6
+- Minimum score is 0.0.
+- Multiple failures compound independently.
+
+## Pass/Fail
+
+- passed = true only if score ≥ 0.7 AND no critical facts are contradicted or key constraints violated.
+
+## Constraints
+
+- Do not assume the response "probably meant" the correct information.
+- Do not excuse retention failures because the response is otherwise helpful.
+- Do not credit partial retention if critical elements are missed.
+- If no prior context is provided, this eval automatically passes with score 1.0.
+- Evaluate only retention of information explicitly stated, not implied.
+
+## Output format (STRICT JSON, one line reason):
+
+{
+  "score": <number between 0.0 and 1.0>,
+  "passed": <true|false>,
+  "metadata": {
+    "reason": "<single concise sentence listing retention failures found or confirming context was correctly maintained>"
+  }
+}`;
+export const knowledgeRetention = createPresetEval({
+    name: 'knowledge-retention',
+    description: 'Evaluates whether response correctly retains context from earlier in conversation',
+    options: {
+        model: openai('gpt-4o'),
+        threshold: 0.7,
+    },
+    handler: async (ctx, input, output, options) => {
+        const prompt = interpolatePrompt(knowledgeRetentionPrompt, {
+            USER_REQUEST: input.request,
+            MODEL_RESPONSE: output.response,
+        });
+        const evaluation = await generateEvalResult({ model: options.model, prompt });
+        return {
+            passed: evaluation.passed && (evaluation.score ?? 1) >= options.threshold,
+            score: evaluation.score,
+            metadata: {
+                ...evaluation.metadata,
+                model: options.model,
+                threshold: options.threshold,
+            },
+        };
+    },
+});
+//# sourceMappingURL=knowledge-retention.js.map

package/dist/knowledge-retention.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledge-retention.js","sourceRoot":"","sources":["../src/knowledge-retention.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAGlB,MAAM,UAAU,CAAC;AAGlB,MAAM,CAAC,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAuDtC,CAAC;AAMH,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAIhD;IACD,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EAAE,mFAAmF;IAChG,OAAO,EAAE;QACR,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;QACvB,SAAS,EAAE,GAAG;KACd;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,wBAAwB,EAAE;YAC1D,YAAY,EAAE,KAAK,CAAC,OAAO;YAC3B,cAAc,EAAE,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9E,OAAO;YACN,MAAM,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS;YACzE,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE;gBACT,GAAG,UAAU,CAAC,QAAQ;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;aAC5B;SACD,CAAC;IACH,CAAC;CACD,CAAC,CAAC"}