@agentuity/evals 0.1.39 → 0.1.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/_utils.d.ts +9 -1
  2. package/dist/_utils.d.ts.map +1 -1
  3. package/dist/_utils.js +21 -2
  4. package/dist/_utils.js.map +1 -1
  5. package/dist/adversarial.d.ts +2 -0
  6. package/dist/adversarial.d.ts.map +1 -1
  7. package/dist/ambiguity.d.ts +2 -0
  8. package/dist/ambiguity.d.ts.map +1 -1
  9. package/dist/answer-completeness.d.ts +2 -0
  10. package/dist/answer-completeness.d.ts.map +1 -1
  11. package/dist/conciseness.d.ts +2 -0
  12. package/dist/conciseness.d.ts.map +1 -1
  13. package/dist/extraneous-content.d.ts +2 -0
  14. package/dist/extraneous-content.d.ts.map +1 -1
  15. package/dist/format.d.ts +2 -0
  16. package/dist/format.d.ts.map +1 -1
  17. package/dist/index.d.ts +1 -1
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/index.js.map +1 -1
  21. package/dist/knowledge-retention.d.ts +2 -0
  22. package/dist/knowledge-retention.d.ts.map +1 -1
  23. package/dist/pii.d.ts +2 -0
  24. package/dist/pii.d.ts.map +1 -1
  25. package/dist/politeness.d.ts +2 -0
  26. package/dist/politeness.d.ts.map +1 -1
  27. package/dist/role-adherence.d.ts +2 -0
  28. package/dist/role-adherence.d.ts.map +1 -1
  29. package/dist/safety.d.ts +2 -0
  30. package/dist/safety.d.ts.map +1 -1
  31. package/dist/self-reference.d.ts +2 -0
  32. package/dist/self-reference.d.ts.map +1 -1
  33. package/dist/types.d.ts +30 -0
  34. package/dist/types.d.ts.map +1 -1
  35. package/dist/types.js.map +1 -1
  36. package/package.json +5 -5
  37. package/src/_utils.ts +33 -3
  38. package/src/index.ts +6 -1
  39. package/src/types.ts +37 -0
package/dist/_utils.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { InferOutput, StandardSchemaV1 } from '@agentuity/core';
2
2
  import { type CreateEvalConfig, type EvalContext, type EvalHandlerResult } from '@agentuity/runtime';
3
- import type { BaseEvalOptions, EvalMiddleware } from './types';
3
+ import type { BaseEvalOptions, EvalLifecycleHooks, EvalMiddleware } from './types';
4
4
  import { type LanguageModel } from 'ai';
5
5
  export declare const DefaultEvalInputSchema: import("@agentuity/schema").ObjectSchema<{
6
6
  request: import("@agentuity/schema").StringSchema;
@@ -48,6 +48,14 @@ type PresetEvalOverrides<TEvalInput extends StandardSchemaV1 | undefined, TEvalO
48
48
  description?: string;
49
49
  } & TOptions> & {
50
50
  middleware?: EvalMiddleware<TAgentInput, TAgentOutput, InferSchemaOutput<TEvalInput>, InferSchemaOutput<TEvalOutput>>;
51
+ /**
52
+ * Called before the eval handler runs.
53
+ */
54
+ onStart?: EvalLifecycleHooks<TAgentInput, TAgentOutput>['onStart'];
55
+ /**
56
+ * Called after the eval handler completes successfully.
57
+ */
58
+ onComplete?: EvalLifecycleHooks<TAgentInput, TAgentOutput>['onComplete'];
51
59
  };
52
60
  type PresetEvalResult<TOptions extends BaseEvalOptions> = CreateEvalConfig<any, any> & {
53
61
  name: string;
package/dist/_utils.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"_utils.d.ts","sourceRoot":"","sources":["../src/_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAEN,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE/D,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAGtD,eAAO,MAAM,sBAAsB;;;EAGjC,CAAC;AACH,eAAO,MAAM,uBAAuB;;EAElC,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,OAAO,sBAAsB,CAAC;AAC7D,MAAM,MAAM,iBAAiB,GAAG,OAAO,uBAAuB,CAAC;AAE/D;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAK7F;AAED,MAAM,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,yBAAyB,GAChC,OAAO,CAAC,iBAAiB,CAAC,CA0B5B;AAID,KAAK,iBAAiB,CAAC,CAAC,IAAI,CAAC,SAAS,gBAAgB,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;AAE9E,KAAK,mBAAmB,CACvB,UAAU,SAAS,gBAAgB,GAAG,SAAS,EAC/C,WAAW,SAAS,gBAAgB,GAAG,SAAS,EAChD,QAAQ,SAAS,eAAe,EAChC,WAAW,EACX,YAAY,IACT,OAAO,CAAC;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,QAAQ,CAAC,GAAG;IACjE,UAAU,CAAC,EAAE,cAAc,CAC1B,WAAW,EACX,YAAY,EACZ,iBAAiB,CAAC,UAAU,CAAC,EAC7B,iBAAiB,CAAC,WAAW,CAAC,CAC9B,CAAC;CACF,CAAC;AAIF,KAAK,gBAAgB,CAAC,QAAQ,SAAS,eAAe,IAAI,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG;IACtF,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,QAAQ,CAAC;CAClB,CAAC;AAEF,wBAAgB,gBAAgB,CAC/B,UAAU,SAAS,gBAAgB,GAAG,SAAS,GAAG,gBAAgB,GAAG,SAAS,EAC9E,WAAW,SAAS,gBAAgB,GAAG,SAAS,GAAG,gBAAgB,GAAG,SAAS,EAC/E,QAAQ,SAAS,eAAe,GAAG,eAAe,EACjD,MAAM,EAAE;IACT,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,CACR,GAAG,EAAE,WAAW,EAChB,KAAK,EAAE,iBAAiB,CAAC,UAAU,CAAC,EACpC,MAAM,EAAE,iBAAiB,CAAC,WAAW,CAAC,EACtC,OAAO,EAAE,QAAQ,KACb,UAAU,CAAC,gBAAgB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACtE,OAAO,EAAE,QAAQ,CAAC;CAClB,GAAG,CAEH,WAAW,SAAS,gBAAgB,GAAG,SAAS,GAAG,GAAG,EAEtD,YAAY,SAAS,gBAAgB,GAAG,SAAS,GAAG,GAAG,EAEvD,SAAS,CAAC,EAAE,mBAAmB,CAC9B,UAAU,EACV,WAAW,EACX,QAAQ,EACR,iBAAiB,CAAC,WAAW,CAAC,EAC9B,iBAAiB,CAAC,YAAY,CAAC,CAC/B,KACG,gBAAgB,CAAC,QAAQ,CAAC,CAsB9B"}
1
+ {"version":3,"file":"_utils.d.ts","sourceRoot":"","sources":["../src/_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAEN,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEnF,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAGtD,eAAO,MAAM,sBAAsB;;;EAGjC,CAAC;AACH,eAAO,MAAM,uBAAuB;;EAElC,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,OAAO,sBAAsB,CAAC;AAC7D,MAAM,MAAM,iBAAiB,GAAG,OAAO,uBAAuB,CAAC;AAE/D;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAK7F;AAED,MAAM,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,yBAAyB,GAChC,OAAO,CAAC,iBAAiB,CAAC,CA0B5B;AAID,KAAK,iBAAiB,CAAC,CAAC,IAAI,CAAC,SAAS,gBAAgB,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;AAE9E,KAAK,mBAAmB,CACvB,UAAU,SAAS,gBAAgB,GAAG,SAAS,EAC/C,WAAW,SAAS,gBAAgB,GAAG,SAAS,EAChD,QAAQ,SAAS,eAAe,EAChC,WAAW,EACX,YAAY,IACT,OAAO,CAAC;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,QAAQ,CAAC,GAAG;IACjE,UAAU,CAAC,EAAE,cAAc,CAC1B,WAAW,EACX,YAAY,EACZ,iBAAiB,CAAC,UAAU,CAAC,EAC7B,iBAAiB,CAAC,WAAW,CAAC,CAC9B,CAAC;IACF;;OAEG;IACH,OAAO,CAAC,EAAE,kBAAkB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,SAAS,CAAC,CAAC;IACnE;;OAEG;IACH,UAAU,CAAC,EAAE,kBAAkB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC;CACzE,CAAC;AAIF,KAAK,gBAAgB,CAAC,QAAQ,SAAS,eAAe,IAAI,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG;IACtF,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,QAAQ,CAAC;CAClB,CAAC;AAEF,wBAAgB,gBAAgB,CAC/B,UAAU,SAAS,gBAAgB,GAAG,SAAS,GAAG,gBAAgB,GAAG,SAAS,EAC9E,WAAW,SAAS,gBAAgB,GAAG,SAAS,GAAG,gBAAgB,GAAG,SAAS,EAC/E,QAAQ,SAAS,eAAe,GAAG,eAAe,EACjD,MAAM,EAAE;IACT,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,CACR,GAAG,EAAE,WAAW,EAChB,KAAK,EAAE,iBAAiB,CAAC,UAAU,CAAC,EACpC,MAAM,EAAE,iBAAiB,CAAC,WAAW,CAAC,EACtC,OAAO,EAAE,QAAQ,KACb,UAAU,CAAC,gBAAgB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACtE,OAAO,EAAE,QAAQ,CAAC;CAClB,GAAG,CAEH,WAAW,SAAS,gBAAgB,GAAG,SAAS,GAAG,GAAG,EAEtD,YAAY,SAAS,gBAAgB,GAAG,SAAS,GAAG,GAAG,EAEvD,SAAS,CAAC,EAAE,mBAAmB,CAC9B,UAAU,EACV,WAAW,EACX,QAAQ,EACR,iBAAiB,CAAC,WAAW,CAAC,EAC9B,iBAAiB,CAAC,YAAY,CAAC,CAC/B,KACG,gBAAgB,CAAC,QAAQ,CAAC,CA4C9B"}
package/dist/_utils.js CHANGED
@@ -59,20 +59,39 @@ export async function generateEvalResult(options) {
59
59
  }
60
60
  export function createPresetEval(config) {
61
61
  return (overrides) => {
62
- const { name, description, middleware, ...optionOverrides } = overrides ?? {};
62
+ const { name, description, middleware, onStart, onComplete, ...optionOverrides } = overrides ?? {};
63
63
  const currentOptions = { ...config.options, ...optionOverrides };
64
64
  return {
65
65
  name: name ?? config.name,
66
66
  description: description ?? config.description,
67
67
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
68
68
  handler: (async (ctx, input, output) => {
69
+ // Call onStart hook if provided
70
+ if (onStart) {
71
+ try {
72
+ await onStart(ctx, input, output);
73
+ }
74
+ catch (error) {
75
+ ctx.logger.warn('Eval onStart hook failed', { error: String(error) });
76
+ }
77
+ }
69
78
  const evalInput = middleware?.transformInput
70
79
  ? middleware.transformInput(input)
71
80
  : input;
72
81
  const evalOutput = middleware?.transformOutput
73
82
  ? middleware.transformOutput(output)
74
83
  : output;
75
- return config.handler(ctx, evalInput, evalOutput, currentOptions);
84
+ const result = await config.handler(ctx, evalInput, evalOutput, currentOptions);
85
+ // Call onComplete hook if provided
86
+ if (onComplete) {
87
+ try {
88
+ await onComplete(ctx, input, output, result);
89
+ }
90
+ catch (error) {
91
+ ctx.logger.warn('Eval onComplete hook failed', { error: String(error) });
92
+ }
93
+ }
94
+ return result;
76
95
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
77
96
  }),
78
97
  options: currentOptions,
package/dist/_utils.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"_utils.js","sourceRoot":"","sources":["../src/_utils.ts"],"names":[],"mappings":"AACA,OAAO,EACN,uBAAuB,GAIvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,mBAAmB,CAAC;AACtC,OAAO,EAAE,YAAY,EAAsB,MAAM,IAAI,CAAC;AAEtD,sEAAsE;AACtE,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,CAAC;AAIH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,SAAiC;IACpF,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CACtC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,EAAE,KAAK,CAAC,EAChE,QAAQ,CACR,CAAC;AACH,CAAC;AAQD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAErD,IAAI,CAAC;YACJ,4DAA4D;YAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3F,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,8BAA8B;YAC9B,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC;gBAAE,MAAM;QACvC,CAAC;IACF,CAAC;IAED,MAAM,IAAI,KAAK,CACd,8CAA8C,UAAU,cAAc,SAAS,EAAE,OAAO,EAAE,CAC1F,CAAC;AACH,CAAC;AA4BD,MAAM,UAAU,gBAAgB,CAI9B,MAUD;IAcA,OAAO,CAAC,SAAS,EAAE,EAAE;QACpB,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QAC9E,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,eAAe,EAAc,CAAC;QAE7E,OAAO;YACN,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI;YACzB,WAAW,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;YAC9C,8DAA8D;YAC9D,OAAO,EAAE,CAAC,KAAK,EAAE,GAAgB,EAAE,KAAU,EAAE,MAAW,EAAE,EAAE;gBAC7D,MAAM,SAAS,GAAG,UAAU,EAAE,cAAc;oBAC3C,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC;oBAClC,CAAC,CAAE,KAAuC,CAAC;gBAC5C,MAAM,UAAU,GAAG,UAAU,EAAE,eAAe;oBAC7C,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;oBACpC,CAAC,CAAE,MAAyC,CAAC;gBAC9C,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;gBAClE,8DAA8D;YAC/D,CAAC,CAAQ;YACT,OAAO,EAAE,cAAc;SACvB,CAAC;IACH,CAAC,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"_utils.js","sourceRoot":"","sources":["../src/_utils.ts"],"names":[],"mappings":"AACA,OAAO,EACN,uBAAuB,GAIvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,mBAAmB,CAAC;AACtC,OAAO,EAAE,YAAY,EAAsB,MAAM,IAAI,CAAC;AAEtD,sEAAsE;AACtE,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,CAAC;AAIH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,SAAiC;IACpF,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CACtC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,EAAE,KAAK,CAAC,EAChE,QAAQ,CACR,CAAC;AACH,CAAC;AAQD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC;IAElC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAErD,IAAI,CAAC;YACJ,4DAA4D;YAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3F,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,8BAA8B;YAC9B,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC;gBAAE,MAAM;QACvC,CAAC;IACF,CAAC;IAED,MAAM,IAAI,KAAK,CACd,8CAA8C,UAAU,cAAc,SAAS,EAAE,OAAO,EAAE,CAC1F,CAAC;AACH,CAAC;AAoCD,MAAM,UAAU,gBAAgB,CAI9B,MAUD;IAcA,OAAO,CAAC,SAAS,EAAE,EAAE;QACpB,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,GAC/E,SAAS,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,eAAe,EAAc,CAAC;QAE7E,OAAO;YACN,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI;YACzB,WAAW,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;YAC9C,8DAA8D;YAC9D,OAAO,EAAE,CAAC,KAAK,EAAE,GAAgB,EAAE,KAAU,EAAE,MAAW,EAAE,EAAE;gBAC7D,gCAAgC;gBAChC,IAAI,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC;wBACJ,MAAM,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;oBACnC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACvE,CAAC;gBACF,CAAC;gBAED,MAAM,SAAS,GAAG,UAAU,EAAE,cAAc;oBAC3C,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC;oBAClC,CAAC,CAAE,KAAuC,CAAC;gBAC5C,MAAM,UAAU,GAAG,UAAU,EAAE,eAAe;oBAC7C,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC;oBACpC,CAAC,CAAE,MAAyC,CAAC;gBAE9C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;gBAEhF,mCAAmC;gBACnC,IAAI,UAAU,EAAE,CAAC;oBAChB,IAAI,CAAC;wBACJ,MAAM,UAAU,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;oBAC9C,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1E,CAAC;gBACF,CAAC;gBAED,OAAO,MAAM,CAAC;gBACd,8DAA8D;YAC/D,CAAC,CAAQ;YACT,OAAO,EAAE,cAAc;SACvB,CAAC;IACH,CAAC,CAAC;AACH,CAAC"}
package/dist/adversarial.d.ts CHANGED
@@ -11,6 +11,8 @@ export declare const adversarial: <TAgentInput extends import("@agentuity/core")
11
11
  }, {
12
12
  response: string;
13
13
  } & {}> | undefined;
14
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
15
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
14
16
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
15
17
  name: string;
16
18
  options: BaseEvalOptions;
package/dist/adversarial.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"adversarial.d.ts","sourceRoot":"","sources":["../src/adversarial.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,iBAAiB,i4EA8C5B,CAAC;AAEH,eAAO,MAAM,WAAW;;;;;;;;;;;;;;CActB,CAAC"}
1
+ {"version":3,"file":"adversarial.d.ts","sourceRoot":"","sources":["../src/adversarial.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,iBAAiB,i4EA8C5B,CAAC;AAEH,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;;CActB,CAAC"}
package/dist/ambiguity.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const ambiguity: <TAgentInput extends import("@agentuity/core").S
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: AmbiguityEvalOptions;
package/dist/ambiguity.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"ambiguity.d.ts","sourceRoot":"","sources":["../src/ambiguity.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,eAAe,2/DA+C1B,CAAC;AAEH,KAAK,oBAAoB,GAAG,eAAe,GAAG;IAC7C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,SAAS;;;;eAHV,MAAM;;;;;;;;;;;;CA4BhB,CAAC"}
1
+ {"version":3,"file":"ambiguity.d.ts","sourceRoot":"","sources":["../src/ambiguity.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,eAAe,2/DA+C1B,CAAC;AAEH,KAAK,oBAAoB,GAAG,eAAe,GAAG;IAC7C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,SAAS;;;;eAHV,MAAM;;;;;;;;;;;;;;CA4BhB,CAAC"}
package/dist/answer-completeness.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const answerCompleteness: <TAgentInput extends import("@agentuity
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: AnswerCompletenessEvalOptions;
package/dist/answer-completeness.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"answer-completeness.d.ts","sourceRoot":"","sources":["../src/answer-completeness.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,wBAAwB,ilEAiDnC,CAAC;AAEH,KAAK,6BAA6B,GAAG,eAAe,GAAG;IACtD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;eAHnB,MAAM;;;;;;;;;;;;CA2BhB,CAAC"}
1
+ {"version":3,"file":"answer-completeness.d.ts","sourceRoot":"","sources":["../src/answer-completeness.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,wBAAwB,ilEAiDnC,CAAC;AAEH,KAAK,6BAA6B,GAAG,eAAe,GAAG;IACtD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;eAHnB,MAAM;;;;;;;;;;;;;;CA2BhB,CAAC"}
package/dist/conciseness.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const conciseness: <TAgentInput extends import("@agentuity/core")
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: ConcisenessEvalOptions;
package/dist/conciseness.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"conciseness.d.ts","sourceRoot":"","sources":["../src/conciseness.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,iBAAiB,g/DAgD5B,CAAC;AAEH,KAAK,sBAAsB,GAAG,eAAe,GAAG;IAC/C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,WAAW;;;;eAHZ,MAAM;;;;;;;;;;;;CA2BhB,CAAC"}
1
+ {"version":3,"file":"conciseness.d.ts","sourceRoot":"","sources":["../src/conciseness.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,iBAAiB,g/DAgD5B,CAAC;AAEH,KAAK,sBAAsB,GAAG,eAAe,GAAG;IAC/C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,WAAW;;;;eAHZ,MAAM;;;;;;;;;;;;;;CA2BhB,CAAC"}
package/dist/extraneous-content.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const extraneousContent: <TAgentInput extends import("@agentuity/
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: ExtraneousContentEvalOptions;
package/dist/extraneous-content.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"extraneous-content.d.ts","sourceRoot":"","sources":["../src/extraneous-content.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,uBAAuB,woEAkDlC,CAAC;AAEH,KAAK,4BAA4B,GAAG,eAAe,GAAG;IACrD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;eAHlB,MAAM;;;;;;;;;;;;CA4BhB,CAAC"}
1
+ {"version":3,"file":"extraneous-content.d.ts","sourceRoot":"","sources":["../src/extraneous-content.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,uBAAuB,woEAkDlC,CAAC;AAEH,KAAK,4BAA4B,GAAG,eAAe,GAAG;IACrD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;eAHlB,MAAM;;;;;;;;;;;;;;CA4BhB,CAAC"}
package/dist/format.d.ts CHANGED
@@ -11,6 +11,8 @@ export declare const format: <TAgentInput extends import("@agentuity/core").Stan
11
11
  }, {
12
12
  response: string;
13
13
  } & {}> | undefined;
14
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
15
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
14
16
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
15
17
  name: string;
16
18
  options: BaseEvalOptions;
package/dist/format.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,YAAY,gjEA4CvB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;CAcjB,CAAC"}
1
+ {"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,YAAY,gjEA4CvB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;CAcjB,CAAC"}
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  export { createPresetEval, interpolatePrompt, generateEvalResult, type DefaultEvalInput, type DefaultEvalOutput, type GenerateEvalResultOptions, } from './_utils';
2
- export { DEFAULT_EVAL_MODEL, type BaseEvalOptions, type EvalMiddleware } from './types';
2
+ export { DEFAULT_EVAL_MODEL, type BaseEvalOptions, type EvalLifecycleHooks, type EvalMiddleware, } from './types';
3
3
  export { politeness, politenessPrompt } from './politeness';
4
4
  export { safety, safetyPrompt } from './safety';
5
5
  export { pii, piiPrompt } from './pii';
package/dist/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC9B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,kBAAkB,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,SAAS,CAAC;AAGxF,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,yBAAyB,GAC9B,MAAM,UAAU,CAAC;AAClB,OAAO,EACN,kBAAkB,EAClB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,cAAc,GACnB,MAAM,SAAS,CAAC;AAGjB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  export { createPresetEval, interpolatePrompt, generateEvalResult, } from './_utils';
2
- export { DEFAULT_EVAL_MODEL } from './types';
2
+ export { DEFAULT_EVAL_MODEL, } from './types';
3
3
  // Evals (each file contains both the prompt and the eval)
4
4
  export { politeness, politenessPrompt } from './politeness';
5
5
  export { safety, safetyPrompt } from './safety';
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAIlB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,kBAAkB,EAA6C,MAAM,SAAS,CAAC;AAExF,0DAA0D;AAC1D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,GAIlB,MAAM,UAAU,CAAC;AAClB,OAAO,EACN,kBAAkB,GAIlB,MAAM,SAAS,CAAC;AAEjB,0DAA0D;AAC1D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}
package/dist/knowledge-retention.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const knowledgeRetention: <TAgentInput extends import("@agentuity
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: KnowledgeRetentionEvalOptions;
package/dist/knowledge-retention.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"knowledge-retention.d.ts","sourceRoot":"","sources":["../src/knowledge-retention.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,wBAAwB,kxEAqDnC,CAAC;AAEH,KAAK,6BAA6B,GAAG,eAAe,GAAG;IACtD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;eAHnB,MAAM;;;;;;;;;;;;CA2BhB,CAAC"}
1
+ {"version":3,"file":"knowledge-retention.d.ts","sourceRoot":"","sources":["../src/knowledge-retention.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,wBAAwB,kxEAqDnC,CAAC;AAEH,KAAK,6BAA6B,GAAG,eAAe,GAAG;IACtD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;eAHnB,MAAM;;;;;;;;;;;;;;CA2BhB,CAAC"}
package/dist/pii.d.ts CHANGED
@@ -11,6 +11,8 @@ export declare const pii: <TAgentInput extends import("@agentuity/core").Standar
11
11
  }, {
12
12
  response: string;
13
13
  } & {}> | undefined;
14
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
15
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
14
16
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
15
17
  name: string;
16
18
  options: BaseEvalOptions;
package/dist/pii.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"pii.d.ts","sourceRoot":"","sources":["../src/pii.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,SAAS,gvDAyCpB,CAAC;AAEH,eAAO,MAAM,GAAG;;;;;;;;;;;;;;CAcd,CAAC"}
1
+ {"version":3,"file":"pii.d.ts","sourceRoot":"","sources":["../src/pii.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,SAAS,gvDAyCpB,CAAC;AAEH,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;CAcd,CAAC"}
package/dist/politeness.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const politeness: <TAgentInput extends import("@agentuity/core").
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: PolitenessEvalOptions;
package/dist/politeness.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"politeness.d.ts","sourceRoot":"","sources":["../src/politeness.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,gBAAgB,ktCAuC3B,CAAC;AAEH,KAAK,qBAAqB,GAAG,eAAe,GAAG;IAC9C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,UAAU;;;;eAHX,MAAM;;;;;;;;;;;;CA2BhB,CAAC"}
1
+ {"version":3,"file":"politeness.d.ts","sourceRoot":"","sources":["../src/politeness.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,gBAAgB,ktCAuC3B,CAAC;AAEH,KAAK,qBAAqB,GAAG,eAAe,GAAG;IAC9C,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,UAAU;;;;eAHX,MAAM;;;;;;;;;;;;;;CA2BhB,CAAC"}
package/dist/role-adherence.d.ts CHANGED
@@ -16,6 +16,8 @@ export declare const roleAdherence: <TAgentInput extends import("@agentuity/core
16
16
  }, {
17
17
  response: string;
18
18
  } & {}> | undefined;
19
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
20
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
19
21
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
20
22
  name: string;
21
23
  options: RoleAdherenceEvalOptions;
package/dist/role-adherence.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"role-adherence.d.ts","sourceRoot":"","sources":["../src/role-adherence.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,mBAAmB,g5EAsD9B,CAAC;AAEH,KAAK,wBAAwB,GAAG,eAAe,GAAG;IACjD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,aAAa;;;;eAHd,MAAM;;;;;;;;;;;;CA2BhB,CAAC"}
1
+ {"version":3,"file":"role-adherence.d.ts","sourceRoot":"","sources":["../src/role-adherence.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,mBAAmB,g5EAsD9B,CAAC;AAEH,KAAK,wBAAwB,GAAG,eAAe,GAAG;IACjD,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,aAAa;;;;eAHd,MAAM;;;;;;;;;;;;;;CA2BhB,CAAC"}
package/dist/safety.d.ts CHANGED
@@ -11,6 +11,8 @@ export declare const safety: <TAgentInput extends import("@agentuity/core").Stan
11
11
  }, {
12
12
  response: string;
13
13
  } & {}> | undefined;
14
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
15
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
14
16
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
15
17
  name: string;
16
18
  options: BaseEvalOptions;
package/dist/safety.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"safety.d.ts","sourceRoot":"","sources":["../src/safety.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,YAAY,+uFA8CvB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;CAcjB,CAAC"}
1
+ {"version":3,"file":"safety.d.ts","sourceRoot":"","sources":["../src/safety.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,YAAY,+uFA8CvB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;CAcjB,CAAC"}
package/dist/self-reference.d.ts CHANGED
@@ -11,6 +11,8 @@ export declare const selfReference: <TAgentInput extends import("@agentuity/core
11
11
  }, {
12
12
  response: string;
13
13
  } & {}> | undefined;
14
+ onStart?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any) => void | Promise<void>) | undefined;
15
+ onComplete?: ((ctx: import("@agentuity/runtime").EvalContext, input: TAgentInput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentInput> : any, output: TAgentOutput extends import("@agentuity/core").StandardSchemaV1<unknown, unknown> ? import("@agentuity/core").InferOutput<TAgentOutput> : any, result: import("@agentuity/runtime").EvalHandlerResult) => void | Promise<void>) | undefined;
14
16
  }) | undefined) => import("@agentuity/runtime").CreateEvalConfig<any, any> & {
15
17
  name: string;
16
18
  options: BaseEvalOptions;
package/dist/self-reference.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"self-reference.d.ts","sourceRoot":"","sources":["../src/self-reference.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,mBAAmB,q3DAoC9B,CAAC;AAEH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;CAgBzB,CAAC"}
1
+ {"version":3,"file":"self-reference.d.ts","sourceRoot":"","sources":["../src/self-reference.ts"],"names":[],"mappings":"AAOA,OAAO,EAAsB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAEnE,eAAO,MAAM,mBAAmB,q3DAoC9B,CAAC;AAEH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;CAgBzB,CAAC"}
package/dist/types.d.ts CHANGED
@@ -1,8 +1,38 @@
1
1
  import type { LanguageModel } from 'ai';
2
+ import type { EvalContext, EvalHandlerResult } from '@agentuity/runtime';
2
3
  export declare const DEFAULT_EVAL_MODEL: LanguageModel;
3
4
  export type BaseEvalOptions = {
4
5
  model: LanguageModel;
5
6
  };
7
+ /**
8
+ * Lifecycle hooks for preset evals.
9
+ * Called before and after the eval handler executes.
10
+ * Hook errors are logged but don't prevent eval execution.
11
+ *
12
+ * @template TInput - The input type passed to hooks (agent's input, not transformed)
13
+ * @template TOutput - The output type passed to hooks (agent's output, not transformed)
14
+ */
15
+ export type EvalLifecycleHooks<TInput = unknown, TOutput = unknown> = {
16
+ /**
17
+ * Called before the eval handler runs.
18
+ * Use for logging, setup, or metrics.
19
+ *
20
+ * @param ctx - Eval context with logger, tracer, storage access
21
+ * @param input - The agent's input (before middleware transform)
22
+ * @param output - The agent's output (before middleware transform)
23
+ */
24
+ onStart?: (ctx: EvalContext, input: TInput, output: TOutput) => void | Promise<void>;
25
+ /**
26
+ * Called after the eval handler completes successfully.
27
+ * Use for logging results, cleanup, or metrics.
28
+ *
29
+ * @param ctx - Eval context with logger, tracer, storage access
30
+ * @param input - The agent's input (before middleware transform)
31
+ * @param output - The agent's output (before middleware transform)
32
+ * @param result - The eval handler's result
33
+ */
34
+ onComplete?: (ctx: EvalContext, input: TInput, output: TOutput, result: EvalHandlerResult) => void | Promise<void>;
35
+ };
6
36
  /**
7
37
  * Middleware to transform agent input/output to preset eval input/output.
8
38
  * Allows reusing preset evals across agents with different schemas.
package/dist/types.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAGxC,eAAO,MAAM,kBAAkB,EAAE,aAA0C,CAAC;AAE5E,MAAM,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,aAAa,CAAC;CACrB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,IAC1E;IACA,cAAc,EAAE,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU,CAAC;IACxD,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,YAAY,KAAK,WAAW,CAAC;CAC5D,GACD;IACA,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU,CAAC;IACzD,eAAe,EAAE,CAAC,WAAW,EAAE,YAAY,KAAK,WAAW,CAAC;CAC3D,CAAC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEzE,eAAO,MAAM,kBAAkB,EAAE,aAA0C,CAAC;AAE5E,MAAM,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,aAAa,CAAC;CACrB,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,IAAI;IACrE;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErF;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,CACZ,GAAG,EAAE,WAAW,EAChB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,iBAAiB,KACrB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,cAAc,CAAC,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,IAC1E;IACA,cAAc,EAAE,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU,CAAC;IACxD,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,YAAY,KAAK,WAAW,CAAC;CAC5D,GACD;IACA,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU,CAAC;IACzD,eAAe,EAAE,CAAC,WAAW,EAAE,YAAY,KAAK,WAAW,CAAC;CAC3D,CAAC"}
package/dist/types.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,MAAM,CAAC,MAAM,kBAAkB,GAAkB,IAAI,CAAC,oBAAoB,CAAC,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpC,MAAM,CAAC,MAAM,kBAAkB,GAAkB,IAAI,CAAC,oBAAoB,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentuity/evals",
3
- "version": "0.1.39",
3
+ "version": "0.1.40",
4
4
  "license": "Apache-2.0",
5
5
  "author": "Agentuity employees and contributors",
6
6
  "type": "module",
@@ -26,14 +26,14 @@
26
26
  "prepublishOnly": "bun run clean && bun run build"
27
27
  },
28
28
  "dependencies": {
29
- "@agentuity/core": "0.1.39",
30
- "@agentuity/runtime": "0.1.39",
31
- "@agentuity/schema": "0.1.39",
29
+ "@agentuity/core": "0.1.40",
30
+ "@agentuity/runtime": "0.1.40",
31
+ "@agentuity/schema": "0.1.40",
32
32
  "@ai-sdk/groq": "latest",
33
33
  "ai": "^6.0.0"
34
34
  },
35
35
  "devDependencies": {
36
- "@agentuity/test-utils": "0.1.39",
36
+ "@agentuity/test-utils": "0.1.40",
37
37
  "@types/bun": "latest",
38
38
  "bun-types": "latest",
39
39
  "typescript": "^5.9.0"
package/src/_utils.ts CHANGED
@@ -5,7 +5,7 @@ import {
5
5
  type EvalContext,
6
6
  type EvalHandlerResult,
7
7
  } from '@agentuity/runtime';
8
- import type { BaseEvalOptions, EvalMiddleware } from './types';
8
+ import type { BaseEvalOptions, EvalLifecycleHooks, EvalMiddleware } from './types';
9
9
  import { s } from '@agentuity/schema';
10
10
  import { generateText, type LanguageModel } from 'ai';
11
11
 
@@ -104,6 +104,14 @@ type PresetEvalOverrides<
104
104
  InferSchemaOutput<TEvalInput>,
105
105
  InferSchemaOutput<TEvalOutput>
106
106
  >;
107
+ /**
108
+ * Called before the eval handler runs.
109
+ */
110
+ onStart?: EvalLifecycleHooks<TAgentInput, TAgentOutput>['onStart'];
111
+ /**
112
+ * Called after the eval handler completes successfully.
113
+ */
114
+ onComplete?: EvalLifecycleHooks<TAgentInput, TAgentOutput>['onComplete'];
107
115
  };
108
116
 
109
117
  // Return type is compatible with any agent's createEval method
@@ -142,7 +150,8 @@ export function createPresetEval<
142
150
  >
143
151
  ) => PresetEvalResult<TOptions> {
144
152
  return (overrides) => {
145
- const { name, description, middleware, ...optionOverrides } = overrides ?? {};
153
+ const { name, description, middleware, onStart, onComplete, ...optionOverrides } =
154
+ overrides ?? {};
146
155
  const currentOptions = { ...config.options, ...optionOverrides } as TOptions;
147
156
 
148
157
  return {
@@ -150,13 +159,34 @@ export function createPresetEval<
150
159
  description: description ?? config.description,
151
160
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
152
161
  handler: (async (ctx: EvalContext, input: any, output: any) => {
162
+ // Call onStart hook if provided
163
+ if (onStart) {
164
+ try {
165
+ await onStart(ctx, input, output);
166
+ } catch (error) {
167
+ ctx.logger.warn('Eval onStart hook failed', { error: String(error) });
168
+ }
169
+ }
170
+
153
171
  const evalInput = middleware?.transformInput
154
172
  ? middleware.transformInput(input)
155
173
  : (input as InferSchemaOutput<TEvalInput>);
156
174
  const evalOutput = middleware?.transformOutput
157
175
  ? middleware.transformOutput(output)
158
176
  : (output as InferSchemaOutput<TEvalOutput>);
159
- return config.handler(ctx, evalInput, evalOutput, currentOptions);
177
+
178
+ const result = await config.handler(ctx, evalInput, evalOutput, currentOptions);
179
+
180
+ // Call onComplete hook if provided
181
+ if (onComplete) {
182
+ try {
183
+ await onComplete(ctx, input, output, result);
184
+ } catch (error) {
185
+ ctx.logger.warn('Eval onComplete hook failed', { error: String(error) });
186
+ }
187
+ }
188
+
189
+ return result;
160
190
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
161
191
  }) as any,
162
192
  options: currentOptions,
package/src/index.ts CHANGED
@@ -6,7 +6,12 @@ export {
6
6
  type DefaultEvalOutput,
7
7
  type GenerateEvalResultOptions,
8
8
  } from './_utils';
9
- export { DEFAULT_EVAL_MODEL, type BaseEvalOptions, type EvalMiddleware } from './types';
9
+ export {
10
+ DEFAULT_EVAL_MODEL,
11
+ type BaseEvalOptions,
12
+ type EvalLifecycleHooks,
13
+ type EvalMiddleware,
14
+ } from './types';
10
15
 
11
16
  // Evals (each file contains both the prompt and the eval)
12
17
  export { politeness, politenessPrompt } from './politeness';
package/src/types.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import type { LanguageModel } from 'ai';
2
2
  import { groq } from '@ai-sdk/groq';
3
+ import type { EvalContext, EvalHandlerResult } from '@agentuity/runtime';
3
4
 
4
5
  export const DEFAULT_EVAL_MODEL: LanguageModel = groq('openai/gpt-oss-20b');
5
6
 
@@ -7,6 +8,42 @@ export type BaseEvalOptions = {
7
8
  model: LanguageModel;
8
9
  };
9
10
 
11
+ /**
12
+ * Lifecycle hooks for preset evals.
13
+ * Called before and after the eval handler executes.
14
+ * Hook errors are logged but don't prevent eval execution.
15
+ *
16
+ * @template TInput - The input type passed to hooks (agent's input, not transformed)
17
+ * @template TOutput - The output type passed to hooks (agent's output, not transformed)
18
+ */
19
+ export type EvalLifecycleHooks<TInput = unknown, TOutput = unknown> = {
20
+ /**
21
+ * Called before the eval handler runs.
22
+ * Use for logging, setup, or metrics.
23
+ *
24
+ * @param ctx - Eval context with logger, tracer, storage access
25
+ * @param input - The agent's input (before middleware transform)
26
+ * @param output - The agent's output (before middleware transform)
27
+ */
28
+ onStart?: (ctx: EvalContext, input: TInput, output: TOutput) => void | Promise<void>;
29
+
30
+ /**
31
+ * Called after the eval handler completes successfully.
32
+ * Use for logging results, cleanup, or metrics.
33
+ *
34
+ * @param ctx - Eval context with logger, tracer, storage access
35
+ * @param input - The agent's input (before middleware transform)
36
+ * @param output - The agent's output (before middleware transform)
37
+ * @param result - The eval handler's result
38
+ */
39
+ onComplete?: (
40
+ ctx: EvalContext,
41
+ input: TInput,
42
+ output: TOutput,
43
+ result: EvalHandlerResult
44
+ ) => void | Promise<void>;
45
+ };
46
+
10
47
  /**
11
48
  * Middleware to transform agent input/output to preset eval input/output.
12
49
  * Allows reusing preset evals across agents with different schemas.