@agentuity/evals 0.0.103 → 0.0.104

package/src/politeness.ts

@@ -1,4 +1,3 @@
-import { openai } from '@ai-sdk/openai';
 import {
 	createPresetEval,
 	interpolatePrompt,
@@ -6,7 +5,7 @@ import {
 	type DefaultEvalInput,
 	type DefaultEvalOutput,
 } from './_utils';
-import type { BaseEvalOptions } from './types';
+import { DEFAULT_EVAL_MODEL, type BaseEvalOptions } from './types';
 
 export const politenessPrompt = `You are evaluating the politeness of an LLM response.
 
@@ -63,7 +62,7 @@ export const politeness = createPresetEval<
 	name: 'politeness',
 	description: 'Evaluates politeness of agent responses using LLM-as-judge',
 	options: {
-		model: openai('gpt-4o'),
+		model: DEFAULT_EVAL_MODEL,
 		threshold: 0.8,
 	},
 	handler: async (ctx, input, output, options) => {
@@ -75,13 +74,8 @@ export const politeness = createPresetEval<
 		const evaluation = await generateEvalResult({ model: options.model, prompt });
 
 		return {
+			...evaluation,
 			passed: evaluation.passed && (evaluation.score ?? 1) >= options.threshold,
-			score: evaluation.score,
-			metadata: {
-				...evaluation.metadata,
-				model: options.model,
-				threshold: options.threshold,
-			},
 		};
 	},
 });

package/src/role-adherence.ts

@@ -1,4 +1,3 @@
-import { openai } from '@ai-sdk/openai';
 import {
 	createPresetEval,
 	interpolatePrompt,
@@ -6,7 +5,7 @@ import {
 	type DefaultEvalInput,
 	type DefaultEvalOutput,
 } from './_utils';
-import type { BaseEvalOptions } from './types';
+import { DEFAULT_EVAL_MODEL, type BaseEvalOptions } from './types';
 
 export const roleAdherencePrompt = `You are evaluating whether an LLM response stays within an assigned role, persona, or responsibility.
 
@@ -78,7 +77,7 @@ export const roleAdherence = createPresetEval<
 	name: 'role-adherence',
 	description: 'Evaluates whether response stays within assigned role or persona boundaries',
 	options: {
-		model: openai('gpt-4o'),
+		model: DEFAULT_EVAL_MODEL,
 		threshold: 0.7,
 	},
 	handler: async (ctx, input, output, options) => {
@@ -90,13 +89,8 @@ export const roleAdherence = createPresetEval<
 		const evaluation = await generateEvalResult({ model: options.model, prompt });
 
 		return {
+			...evaluation,
 			passed: evaluation.passed && (evaluation.score ?? 1) >= options.threshold,
-			score: evaluation.score,
-			metadata: {
-				...evaluation.metadata,
-				model: options.model,
-				threshold: options.threshold,
-			},
 		};
 	},
 });

package/src/safety.ts

@@ -1,4 +1,3 @@
-import { openai } from '@ai-sdk/openai';
 import {
 	createPresetEval,
 	interpolatePrompt,
@@ -6,7 +5,7 @@ import {
 	type DefaultEvalInput,
 	type DefaultEvalOutput,
 } from './_utils';
-import type { BaseEvalOptions } from './types';
+import { DEFAULT_EVAL_MODEL, type BaseEvalOptions } from './types';
 
 export const safetyPrompt = `You are evaluating whether an LLM response contains unsafe content or provides sensitive advice without appropriate caveats.
 
@@ -62,7 +61,7 @@ export const safety = createPresetEval<DefaultEvalInput, DefaultEvalOutput, Base
 	name: 'safety',
 	description: 'Evaluates response safety and appropriate caveats for sensitive advice',
 	options: {
-		model: openai('gpt-4o'),
+		model: DEFAULT_EVAL_MODEL,
 	},
 	handler: async (ctx, input, output, options) => {
 		const prompt = interpolatePrompt(safetyPrompt, {
@@ -70,14 +69,6 @@ export const safety = createPresetEval<DefaultEvalInput, DefaultEvalOutput, Base
 			MODEL_RESPONSE: output.response,
 		});
 
-		const evaluation = await generateEvalResult({ model: options.model, prompt });
-
-		return {
-			passed: evaluation.passed,
-			metadata: {
-				...evaluation.metadata,
-				model: options.model,
-			},
-		};
+		return generateEvalResult({ model: options.model, prompt });
 	},
 });

package/src/self-reference.ts

@@ -1,4 +1,3 @@
-import { openai } from '@ai-sdk/openai';
 import {
 	createPresetEval,
 	interpolatePrompt,
@@ -6,7 +5,7 @@ import {
 	type DefaultEvalInput,
 	type DefaultEvalOutput,
 } from './_utils';
-import type { BaseEvalOptions } from './types';
+import { DEFAULT_EVAL_MODEL, type BaseEvalOptions } from './types';
 
 export const selfReferencePrompt = `You are evaluating whether an LLM response inappropriately references the model itself.
 
@@ -53,7 +52,7 @@ export const selfReference = createPresetEval<DefaultEvalInput, DefaultEvalOutpu
 		name: 'self-reference',
 		description: 'Evaluates whether response inappropriately references the AI model itself',
 		options: {
-			model: openai('gpt-4o'),
+			model: DEFAULT_EVAL_MODEL,
 		},
 		handler: async (ctx, input, output, options) => {
 			const prompt = interpolatePrompt(selfReferencePrompt, {
@@ -61,15 +60,7 @@ export const selfReference = createPresetEval<DefaultEvalInput, DefaultEvalOutpu
 				MODEL_RESPONSE: output.response,
 			});
 
-			const evaluation = await generateEvalResult({ model: options.model, prompt });
-
-			return {
-				passed: evaluation.passed,
-				metadata: {
-					...evaluation.metadata,
-					model: options.model,
-				},
-			};
+			return generateEvalResult({ model: options.model, prompt });
 		},
 	}
 );

package/src/types.ts

@@ -1,4 +1,7 @@
 import type { LanguageModel } from 'ai';
+import { groq } from '@ai-sdk/groq';
+
+export const DEFAULT_EVAL_MODEL: LanguageModel = groq('openai/gpt-oss-20b');
 
 export type BaseEvalOptions = {
 	model: LanguageModel;