@kilnai/core 0.9.2 → 0.13.0

package/dist/eval/experiment-runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-runner.js","sourceRoot":"","sources":["../../src/eval/experiment-runner.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAGlF,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAiBhD,MAAM,OAAO,gBAAgB;IACE;IAA7B,YAA6B,MAA8B;QAA9B,WAAM,GAAN,MAAM,CAAwB;IAAG,CAAC;IAE/D,KAAK,CAAC,GAAG;QACP,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAuB,EAAE,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE/D,MAAM,SAAS,GAAc;gBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,OAAO,EAAE,SAAS,CAAC,OAAO;aAC3B,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,CAAC;oBACH,OAAO,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO;wBACL,IAAI,EAAE,CAAC,CAAC,IAAI;wBACZ,KAAK,EAAE,CAAC;wBACR,SAAS,EAAE,GAAG,YAAY,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;qBAChE,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,MAAM;gBACN,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,UAAU,EAAE;oBACV,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,YAAY,EAAE,SAAS,CAAC,YAAY;iBACrC;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI;YACrC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,OAAO;YACP,SAAS;YACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"experiment-runner.js","sourceRoot":"","sources":["../../src/eval/experiment-runner.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAGlF,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAiBhD,MAAM,OAAO,gBAAgB;IACE;IAA7B,YAA6B,MAA8B;QAA9B,WAAM,GAAN,MAAM,CAAwB;IAAG,CAAC;IAE/D,KAAK,CAAC,GAAG;QACP,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAuB,EAAE,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE/D,MAAM,SAAS,GAAc;gBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,CAAC;oBACH,OAAO,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAClC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO;wBACL,IAAI,EAAE,CAAC,CAAC,IAAI;wBACZ,KAAK,EAAE,CAAC;wBACR,SAAS,EAAE,GAAG,YAAY,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;qBAChE,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,MAAM;gBACN,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,UAAU,EAAE;oBACV,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,YAAY,EAAE,SAAS,CAAC,YAAY;iBACrC;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI;YACrC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/C,OAAO;YACP,SAAS;YACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC,CAAC;IACJ,CAAC;CACF"}

package/dist/eval/index.d.ts

@@ -5,6 +5,8 @@ export { JsonValidityScorer } from "./scorers/json-validity-scorer.js";
 export { LengthScorer } from "./scorers/length-scorer.js";
 export { LatencyScorer } from "./scorers/latency-scorer.js";
 export { CostScorer } from "./scorers/cost-scorer.js";
+export { EffortScorer } from "./scorers/effort-scorer.js";
+export { ResolutionScorer } from "./scorers/resolution-scorer.js";
 export { CompositeScorer } from "./scorers/composite-scorer.js";
 export { FaithfulnessScorer } from "./scorers/faithfulness-scorer.js";
 export { RelevanceScorer } from "./scorers/relevance-scorer.js";
@@ -12,10 +14,21 @@ export { CoherenceScorer } from "./scorers/coherence-scorer.js";
 export { HallucinationScorer } from "./scorers/hallucination-scorer.js";
 export { ToxicityScorer } from "./scorers/toxicity-scorer.js";
 export { CustomPromptScorer } from "./scorers/custom-prompt-scorer.js";
+export { PolicyAdherenceScorer } from "./scorers/policy-adherence-scorer.js";
+export { ContextRelevanceScorer } from "./scorers/context-relevance-scorer.js";
+export { ToolTrajectoryScorer } from "./scorers/tool-trajectory-scorer.js";
+export { ToolCallingAccuracyScorer } from "./scorers/tool-calling-accuracy-scorer.js";
+export { MultiTurnConsistencyScorer } from "./scorers/multi-turn-consistency-scorer.js";
+export { SafetyPreservationScorer } from "./scorers/safety-preservation-scorer.js";
+export { RoutingAccuracyScorer } from "./scorers/routing-accuracy-scorer.js";
+export { HandoffQualityScorer } from "./scorers/handoff-quality-scorer.js";
+export { MilestoneScorer } from "./scorers/milestone-scorer.js";
 export { parseDatasetJsonl } from "./dataset-loader.js";
 export { createScorer } from "./scorer-factory.js";
 export { ExperimentRunner } from "./experiment-runner.js";
 export type { ExperimentRunnerConfig, GenerateOutputResult } from "./experiment-runner.js";
 export { compareExperiments } from "./experiment-comparator.js";
 export type { ComparisonResult, ScorerComparison } from "./experiment-comparator.js";
+export { ConsistencyRunner } from "./consistency-runner.js";
+export type { ConsistencyRunnerConfig, ConsistencyResult, ConsistencyItemResult } from "./consistency-runner.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAEA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACpJ,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,YAAY,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAEA,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACpJ,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAC3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,2CAA2C,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,4CAA4C,CAAC;AACxF,OAAO,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,YAAY,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC"}

package/dist/eval/index.js

@@ -5,6 +5,8 @@ export { JsonValidityScorer } from "./scorers/json-validity-scorer.js";
 export { LengthScorer } from "./scorers/length-scorer.js";
 export { LatencyScorer } from "./scorers/latency-scorer.js";
 export { CostScorer } from "./scorers/cost-scorer.js";
+export { EffortScorer } from "./scorers/effort-scorer.js";
+export { ResolutionScorer } from "./scorers/resolution-scorer.js";
 export { CompositeScorer } from "./scorers/composite-scorer.js";
 export { FaithfulnessScorer } from "./scorers/faithfulness-scorer.js";
 export { RelevanceScorer } from "./scorers/relevance-scorer.js";
@@ -12,8 +14,18 @@ export { CoherenceScorer } from "./scorers/coherence-scorer.js";
 export { HallucinationScorer } from "./scorers/hallucination-scorer.js";
 export { ToxicityScorer } from "./scorers/toxicity-scorer.js";
 export { CustomPromptScorer } from "./scorers/custom-prompt-scorer.js";
+export { PolicyAdherenceScorer } from "./scorers/policy-adherence-scorer.js";
+export { ContextRelevanceScorer } from "./scorers/context-relevance-scorer.js";
+export { ToolTrajectoryScorer } from "./scorers/tool-trajectory-scorer.js";
+export { ToolCallingAccuracyScorer } from "./scorers/tool-calling-accuracy-scorer.js";
+export { MultiTurnConsistencyScorer } from "./scorers/multi-turn-consistency-scorer.js";
+export { SafetyPreservationScorer } from "./scorers/safety-preservation-scorer.js";
+export { RoutingAccuracyScorer } from "./scorers/routing-accuracy-scorer.js";
+export { HandoffQualityScorer } from "./scorers/handoff-quality-scorer.js";
+export { MilestoneScorer } from "./scorers/milestone-scorer.js";
 export { parseDatasetJsonl } from "./dataset-loader.js";
 export { createScorer } from "./scorer-factory.js";
 export { ExperimentRunner } from "./experiment-runner.js";
 export { compareExperiments } from "./experiment-comparator.js";
+export { ConsistencyRunner } from "./consistency-runner.js";
 //# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAG/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAG/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAC3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,2CAA2C,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,4CAA4C,CAAC;AACxF,OAAO,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC"}

package/dist/eval/scorer-factory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scorer-factory.d.ts","sourceRoot":"","sources":["../../src/eval/scorer-factory.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAgBxE,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,CAAC,EAAE,SAAS,GAAG,MAAM,CA8B9E"}
+{"version":3,"file":"scorer-factory.d.ts","sourceRoot":"","sources":["../../src/eval/scorer-factory.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AA2BxE,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,CAAC,EAAE,SAAS,GAAG,MAAM,CA8C9E"}

package/dist/eval/scorer-factory.js

@@ -13,6 +13,17 @@ import { CoherenceScorer } from "./scorers/coherence-scorer.js";
 import { HallucinationScorer } from "./scorers/hallucination-scorer.js";
 import { ToxicityScorer } from "./scorers/toxicity-scorer.js";
 import { CustomPromptScorer } from "./scorers/custom-prompt-scorer.js";
+import { PolicyAdherenceScorer } from "./scorers/policy-adherence-scorer.js";
+import { ContextRelevanceScorer } from "./scorers/context-relevance-scorer.js";
+import { ToolTrajectoryScorer } from "./scorers/tool-trajectory-scorer.js";
+import { EffortScorer } from "./scorers/effort-scorer.js";
+import { ResolutionScorer } from "./scorers/resolution-scorer.js";
+import { ToolCallingAccuracyScorer } from "./scorers/tool-calling-accuracy-scorer.js";
+import { MultiTurnConsistencyScorer } from "./scorers/multi-turn-consistency-scorer.js";
+import { SafetyPreservationScorer } from "./scorers/safety-preservation-scorer.js";
+import { RoutingAccuracyScorer } from "./scorers/routing-accuracy-scorer.js";
+import { HandoffQualityScorer } from "./scorers/handoff-quality-scorer.js";
+import { MilestoneScorer } from "./scorers/milestone-scorer.js";
 export function createScorer(config, llm) {
     switch (config.type) {
         case "exact-match":
@@ -27,6 +38,16 @@ export function createScorer(config, llm) {
             return new LatencyScorer(config.maxLatencyMs ?? 5000);
         case "cost":
             return new CostScorer(config.maxCostUsd ?? 1.0);
+        case "effort":
+            return new EffortScorer();
+        case "resolution":
+            return new ResolutionScorer();
+        case "tool-calling-accuracy":
+            return new ToolCallingAccuracyScorer();
+        case "routing-accuracy":
+            return new RoutingAccuracyScorer();
+        case "milestone":
+            return new MilestoneScorer();
         case "composite": {
             const subScorers = (config.scorers ?? []).map((s) => createScorer(s, llm));
             return new CompositeScorer(config.name, subScorers);
@@ -37,6 +58,12 @@ export function createScorer(config, llm) {
         case "hallucination":
         case "toxicity":
         case "custom-prompt":
+        case "policy-adherence":
+        case "context-relevance":
+        case "tool-trajectory":
+        case "multi-turn-consistency":
+        case "safety-preservation":
+        case "handoff-quality":
             return createLLMScorer(config, llm);
         default:
             throw new KilnError("EVAL_SCORER_FAILED", `Unknown scorer type: ${config.type}`, {
@@ -58,6 +85,12 @@ function createLLMScorer(config, llm) {
         case "hallucination": return new HallucinationScorer(llm);
         case "toxicity": return new ToxicityScorer(llm);
         case "custom-prompt": return new CustomPromptScorer(config.name, config.prompt ?? "", llm);
+        case "policy-adherence": return new PolicyAdherenceScorer(llm, config.policies ?? []);
+        case "context-relevance": return new ContextRelevanceScorer(llm);
+        case "tool-trajectory": return new ToolTrajectoryScorer(llm);
+        case "multi-turn-consistency": return new MultiTurnConsistencyScorer(llm);
+        case "safety-preservation": return new SafetyPreservationScorer(llm);
+        case "handoff-quality": return new HandoffQualityScorer(llm);
         default:
             throw new KilnError("EVAL_SCORER_FAILED", `Unknown LLM scorer type: ${config.type}`, {
                 context: { type: config.type, name: config.name },

package/dist/eval/scorer-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"scorer-factory.js","sourceRoot":"","sources":["../../src/eval/scorer-factory.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAI3D,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAEvE,MAAM,UAAU,YAAY,CAAC,MAAwB,EAAE,GAAe;IACpE,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,aAAa;YAChB,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,eAAe;YAClB,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,KAAK,QAAQ;YACX,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9D,KAAK,SAAS;YACZ,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;QACxD,KAAK,MAAM;YACT,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC;QAClD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,MAAM,UAAU,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3E,OAAO,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACtD,CAAC;QACD,KAAK,cAAc,CAAC;QACpB,KAAK,WAAW,CAAC;QACjB,KAAK,WAAW,CAAC;QACjB,KAAK,eAAe,CAAC;QACrB,KAAK,UAAU,CAAC;QAChB,KAAK,eAAe;YAClB,OAAO,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC;YACE,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,wBAAwB,MAAM,CAAC,IAAI,EAAE,EAAE;gBAC/E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;aAClD,CAAC,CAAC;IACP,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAwB,EAAE,GAAe;IAChE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,eAAe,MAAM,CAAC,IAAI,iCAAiC,EAAE;YACrG,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;YACjD,UAAU,EAAE,2DAA2D;SACxE,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,cAAc,CAAC,CAAC,OAAO,IAAI,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACxD,KAAK,WAAW,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAClD,KAAK,WAAW,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAClD,KAAK,eAAe,CAAC,CAAC,OAAO,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAC1D,KAAK,UAAU,CAAC,CAAC,OAAO,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC;QAChD,KAAK,eAAe,CAAC,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3F;YACE,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,4BAA4B,MAAM,CAAC,IAAI,EAAE,EAAE;gBACnF,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;aAClD,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}
+{"version":3,"file":"scorer-factory.js","sourceRoot":"","sources":["../../src/eval/scorer-factory.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAI3D,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,yBAAyB,EAAE,MAAM,2CAA2C,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,4CAA4C,CAAC;AACxF,OAAO,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAEhE,MAAM,UAAU,YAAY,CAAC,MAAwB,EAAE,GAAe;IACpE,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,aAAa;YAChB,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QACrD,KAAK,eAAe;YAClB,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,KAAK,QAAQ;YACX,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9D,KAAK,SAAS;YACZ,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;QACxD,KAAK,MAAM;YACT,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC;QAClD,KAAK,QAAQ;YACX,OAAO,IAAI,YAAY,EAAE,CAAC;QAC5B,KAAK,YAAY;YACf,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,KAAK,uBAAuB;YAC1B,OAAO,IAAI,yBAAyB,EAAE,CAAC;QACzC,KAAK,kBAAkB;YACrB,OAAO,IAAI,qBAAqB,EAAE,CAAC;QACrC,KAAK,WAAW;YACd,OAAO,IAAI,eAAe,EAAE,CAAC;QAC/B,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,MAAM,UAAU,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3E,OAAO,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACtD,CAAC;QACD,KAAK,cAAc,CAAC;QACpB,KAAK,WAAW,CAAC;QACjB,KAAK,WAAW,CAAC;QACjB,KAAK,eAAe,CAAC;QACrB,KAAK,UAAU,CAAC;QAChB,KAAK,eAAe,CAAC;QACrB,KAAK,kBAAkB,CAAC;QACxB,KAAK,mBAAmB,CAAC;QACzB,KAAK,iBAAiB,CAAC;QACvB,KAAK,wBAAwB,CAAC;QAC9B,KAAK,qBAAqB,CAAC;QAC3B,KAAK,iBAAiB;YACpB,OAAO,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtC;YACE,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,wBAAwB,MAAM,CAAC,IAAI,EAAE,EAAE;gBAC/E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;aAClD,CAAC,CAAC;IACP,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAwB,EAAE,GAAe;IAChE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,eAAe,MAAM,CAAC,IAAI,iCAAiC,EAAE;YACrG,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;YACjD,UAAU,EAAE,2DAA2D;SACxE,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,cAAc,CAAC,CAAC,OAAO,IAAI,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACxD,KAAK,WAAW,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAClD,KAAK,WAAW,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAClD,KAAK,eAAe,CAAC,CAAC,OAAO,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAC1D,KAAK,UAAU,CAAC,CAAC,OAAO,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC;QAChD,KAAK,eAAe,CAAC,CAAC,OAAO,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3F,KAAK,kBAAkB,CAAC,CAAC,OAAO,IAAI,qBAAqB,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QACtF,KAAK,mBAAmB,CAAC,CAAC,OAAO,IAAI,sBAAsB,CAAC,GAAG,CAAC,CAAC;QACjE,KAAK,iBAAiB,CAAC,CAAC,OAAO,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC7D,KAAK,wBAAwB,CAAC,CAAC,OAAO,IAAI,0BAA0B,CAAC,GAAG,CAAC,CAAC;QAC1E,KAAK,qBAAqB,CAAC,CAAC,OAAO,IAAI,wBAAwB,CAAC,GAAG,CAAC,CAAC;QACrE,KAAK,iBAAiB,CAAC,CAAC,OAAO,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC7D;YACE,MAAM,IAAI,SAAS,CAAC,oBAAoB,EAAE,4BAA4B,MAAM,CAAC,IAAI,EAAE,EAAE;gBACnF,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;aAClD,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}

package/dist/eval/scorers/context-relevance-scorer.d.ts

@@ -0,0 +1,8 @@
+import type { EvalInput, EvalScore, Scorer, ScorerLLM } from "../types.js";
+export declare class ContextRelevanceScorer implements Scorer {
+    private readonly llm;
+    readonly name = "context-relevance";
+    constructor(llm: ScorerLLM);
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=context-relevance-scorer.d.ts.map

package/dist/eval/scorers/context-relevance-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-relevance-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/context-relevance-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAG3E,qBAAa,sBAAuB,YAAW,MAAM;IAGvC,OAAO,CAAC,QAAQ,CAAC,GAAG;IAFhC,QAAQ,CAAC,IAAI,uBAAuB;gBAEP,GAAG,EAAE,SAAS;IAErC,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAwBlD"}

package/dist/eval/scorers/context-relevance-scorer.js

@@ -0,0 +1,32 @@
+// ContextRelevanceScorer: evaluates whether retrieved context chunks are relevant to the input query
+import { parseLLMResponse } from "./parse-llm-response.js";
+export class ContextRelevanceScorer {
+    llm;
+    name = "context-relevance";
+    constructor(llm) {
+        this.llm = llm;
+    }
+    async score(input) {
+        const chunks = input.context ?? [];
+        if (chunks.length === 0) {
+            return { name: this.name, score: 0, reasoning: "No context provided" };
+        }
+        const contextList = chunks.map((c, i) => `[${i + 1}] ${c}`).join("\n\n");
+        const prompt = `Evaluate context relevance. Are the retrieved context chunks relevant to the user's query? This measures retrieval quality, not answer quality.
+
+Query: ${input.input}
+
+Retrieved context:
+${contextList}
+
+Score 1.0 if all chunks are highly relevant to the query. Score 0.0 if none are relevant. Use intermediate scores based on the proportion and degree of relevance.
+
+Respond EXACTLY in this format:
+SCORE: <number from 0.0 to 1.0>
+REASONING: <one sentence explanation>`;
+        const response = await this.llm.evaluate(prompt);
+        const { score, reasoning } = parseLLMResponse(response, this.name);
+        return { name: this.name, score, reasoning };
+    }
+}
+//# sourceMappingURL=context-relevance-scorer.js.map

package/dist/eval/scorers/context-relevance-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-relevance-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/context-relevance-scorer.ts"],"names":[],"mappings":"AAAA,qGAAqG;AAGrG,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,MAAM,OAAO,sBAAsB;IAGJ;IAFpB,IAAI,GAAG,mBAAmB,CAAC;IAEpC,YAA6B,GAAc;QAAd,QAAG,GAAH,GAAG,CAAW;IAAG,CAAC;IAE/C,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,qBAAqB,EAAE,CAAC;QACzE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG;;SAEV,KAAK,CAAC,KAAK;;;EAGlB,WAAW;;;;;;sCAMyB,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACnE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC/C,CAAC;CACF"}

package/dist/eval/scorers/effort-scorer.d.ts

@@ -0,0 +1,6 @@
+import type { EvalInput, EvalScore, Scorer } from "../types.js";
+export declare class EffortScorer implements Scorer {
+    readonly name = "effort";
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=effort-scorer.d.ts.map

package/dist/eval/scorers/effort-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"effort-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/effort-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIhE,qBAAa,YAAa,YAAW,MAAM;IACzC,QAAQ,CAAC,IAAI,YAAY;IAEnB,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CASlD"}

package/dist/eval/scorers/effort-scorer.js

@@ -0,0 +1,15 @@
+// EffortScorer: rule-based scorer bridging enrichment effort score into eval framework
+import { computeEffortScore } from "../../enrichment/effort-score.js";
+export class EffortScorer {
+    name = "effort";
+    async score(input) {
+        const components = input.metadata?.effortComponents;
+        if (!components) {
+            return { name: this.name, score: 0, reasoning: "No effort components in metadata" };
+        }
+        const rawScore = computeEffortScore(components);
+        const normalized = rawScore / 10; // 0-10 -> 0-1
+        return { name: this.name, score: normalized, reasoning: `Effort score: ${rawScore}/10` };
+    }
+}
+//# sourceMappingURL=effort-scorer.js.map

package/dist/eval/scorers/effort-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"effort-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/effort-scorer.ts"],"names":[],"mappings":"AAAA,uFAAuF;AAGvF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AAGtE,MAAM,OAAO,YAAY;IACd,IAAI,GAAG,QAAQ,CAAC;IAEzB,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,gBAAgD,CAAC;QACpF,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,kCAAkC,EAAE,CAAC;QACtF,CAAC;QACD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,GAAG,EAAE,CAAC,CAAC,cAAc;QAChD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,QAAQ,KAAK,EAAE,CAAC;IAC3F,CAAC;CACF"}

package/dist/eval/scorers/handoff-quality-scorer.d.ts

@@ -0,0 +1,8 @@
+import type { EvalInput, EvalScore, Scorer, ScorerLLM } from "../types.js";
+export declare class HandoffQualityScorer implements Scorer {
+    private readonly llm;
+    readonly name = "handoff-quality";
+    constructor(llm: ScorerLLM);
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=handoff-quality-scorer.d.ts.map

package/dist/eval/scorers/handoff-quality-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handoff-quality-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/handoff-quality-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AA4B3E,qBAAa,oBAAqB,YAAW,MAAM;IAGrC,OAAO,CAAC,QAAQ,CAAC,GAAG;IAFhC,QAAQ,CAAC,IAAI,qBAAqB;gBAEL,GAAG,EAAE,SAAS;IAErC,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAuClD"}

package/dist/eval/scorers/handoff-quality-scorer.js

@@ -0,0 +1,65 @@
+// HandoffQualityScorer: LLM-as-judge for context preservation across agent handoffs
+import { parseLLMResponse } from "./parse-llm-response.js";
+function extractHandoffs(metadata) {
+    if (!metadata)
+        return undefined;
+    const raw = metadata["handoffHistory"];
+    if (!Array.isArray(raw))
+        return undefined;
+    const events = [];
+    for (const entry of raw) {
+        if (typeof entry === "object" &&
+            entry !== null &&
+            typeof entry["fromAgent"] === "string" &&
+            typeof entry["toAgent"] === "string") {
+            events.push(entry);
+        }
+    }
+    return events.length > 0 ? events : undefined;
+}
+export class HandoffQualityScorer {
+    llm;
+    name = "handoff-quality";
+    constructor(llm) {
+        this.llm = llm;
+    }
+    async score(input) {
+        const handoffs = extractHandoffs(input.metadata);
+        if (!handoffs) {
+            return { name: this.name, score: 0, reasoning: "No handoff history in metadata" };
+        }
+        const handoffLog = handoffs
+            .map((h, i) => {
+            const parts = [`Handoff ${i + 1}: ${h.fromAgent} -> ${h.toAgent}`];
+            if (h.reason)
+                parts.push(`  Reason: ${h.reason}`);
+            if (h.summary)
+                parts.push(`  Summary: ${h.summary}`);
+            return parts.join("\n");
+        })
+            .join("\n\n");
+        const prompt = `Evaluate the quality of agent handoffs in this conversation. Was context preserved across each agent switch?
+
+User query: ${input.input}
+Final output: ${input.output}
+
+Handoff history:
+${handoffLog}
+
+Evaluate:
+1. Was the handoff reason appropriate (correct agent for the task)?
+2. Was the context summary accurate and complete?
+3. Did the receiving agent pick up seamlessly without re-asking for information?
+4. Was any critical context lost during the handoff?
+
+Score 1.0 for seamless handoffs with full context preservation. Score 0.0 for complete context loss.
+
+Respond EXACTLY in this format:
+SCORE: <number from 0.0 to 1.0>
+REASONING: <one sentence explanation>`;
+        const response = await this.llm.evaluate(prompt);
+        const { score, reasoning } = parseLLMResponse(response, this.name);
+        return { name: this.name, score, reasoning };
+    }
+}
+//# sourceMappingURL=handoff-quality-scorer.js.map

package/dist/eval/scorers/handoff-quality-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handoff-quality-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/handoff-quality-scorer.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAGpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAS3D,SAAS,eAAe,CAAC,QAA6C;IACpE,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QACxB,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,OAAQ,KAAiC,CAAC,WAAW,CAAC,KAAK,QAAQ;YACnE,OAAQ,KAAiC,CAAC,SAAS,CAAC,KAAK,QAAQ,EACjE,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAqB,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC;AAED,MAAM,OAAO,oBAAoB;IAGF;IAFpB,IAAI,GAAG,iBAAiB,CAAC;IAElC,YAA6B,GAAc;QAAd,QAAG,GAAH,GAAG,CAAW;IAAG,CAAC;IAE/C,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,gCAAgC,EAAE,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAG,QAAQ;aACxB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACZ,MAAM,KAAK,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,SAAS,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,IAAI,CAAC,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAClD,IAAI,CAAC,CAAC,OAAO;gBAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,MAAM,GAAG;;cAEL,KAAK,CAAC,KAAK;gBACT,KAAK,CAAC,MAAM;;;EAG1B,UAAU;;;;;;;;;;;;sCAY0B,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACnE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC/C,CAAC;CACF"}

package/dist/eval/scorers/milestone-scorer.d.ts

@@ -0,0 +1,6 @@
+import type { EvalInput, EvalScore, Scorer } from "../types.js";
+export declare class MilestoneScorer implements Scorer {
+    readonly name = "milestone";
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=milestone-scorer.d.ts.map

package/dist/eval/scorers/milestone-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"milestone-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/milestone-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyBhE,qBAAa,eAAgB,YAAW,MAAM;IAC5C,QAAQ,CAAC,IAAI,eAAe;IAEtB,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAelD"}

package/dist/eval/scorers/milestone-scorer.js

@@ -0,0 +1,35 @@
+// MilestoneScorer: rule-based scorer tracking intermediate checkpoint achievement
+function extractMilestones(metadata) {
+    if (!metadata)
+        return undefined;
+    const raw = metadata["milestones"];
+    if (!Array.isArray(raw))
+        return undefined;
+    const milestones = [];
+    for (const entry of raw) {
+        if (typeof entry === "object" &&
+            entry !== null &&
+            typeof entry["name"] === "string" &&
+            typeof entry["completed"] === "boolean") {
+            milestones.push(entry);
+        }
+    }
+    return milestones.length > 0 ? milestones : undefined;
+}
+export class MilestoneScorer {
+    name = "milestone";
+    async score(input) {
+        const milestones = extractMilestones(input.metadata);
+        if (!milestones) {
+            return { name: this.name, score: 0, reasoning: "No milestones in metadata" };
+        }
+        const completed = milestones.filter((m) => m.completed);
+        const score = completed.length / milestones.length;
+        const missed = milestones.filter((m) => !m.completed).map((m) => m.name);
+        const parts = [`${completed.length}/${milestones.length} milestones completed`];
+        if (missed.length > 0)
+            parts.push(`missed: ${missed.join(", ")}`);
+        return { name: this.name, score: Math.round(score * 100) / 100, reasoning: parts.join("; ") };
+    }
+}
+//# sourceMappingURL=milestone-scorer.js.map

package/dist/eval/scorers/milestone-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"milestone-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/milestone-scorer.ts"],"names":[],"mappings":"AAAA,kFAAkF;AASlF,SAAS,iBAAiB,CAAC,QAA6C;IACtE,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QACxB,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,OAAQ,KAAiC,CAAC,MAAM,CAAC,KAAK,QAAQ;YAC9D,OAAQ,KAAiC,CAAC,WAAW,CAAC,KAAK,SAAS,EACpE,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,KAAkB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACxD,CAAC;AAED,MAAM,OAAO,eAAe;IACjB,IAAI,GAAG,WAAW,CAAC;IAE5B,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC;QAC/E,CAAC;QAED,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;QACnD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,uBAAuB,CAAC,CAAC;QAChF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAChG,CAAC;CACF"}

package/dist/eval/scorers/multi-turn-consistency-scorer.d.ts

@@ -0,0 +1,8 @@
+import type { EvalInput, EvalScore, Scorer, ScorerLLM } from "../types.js";
+export declare class MultiTurnConsistencyScorer implements Scorer {
+    private readonly llm;
+    readonly name = "multi-turn-consistency";
+    constructor(llm: ScorerLLM);
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=multi-turn-consistency-scorer.d.ts.map

package/dist/eval/scorers/multi-turn-consistency-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi-turn-consistency-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/multi-turn-consistency-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AA0B3E,qBAAa,0BAA2B,YAAW,MAAM;IAG3C,OAAO,CAAC,QAAQ,CAAC,GAAG;IAFhC,QAAQ,CAAC,IAAI,4BAA4B;gBAEZ,GAAG,EAAE,SAAS;IAErC,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CA+BlD"}

package/dist/eval/scorers/multi-turn-consistency-scorer.js

@@ -0,0 +1,55 @@
+// MultiTurnConsistencyScorer: LLM-as-judge for context retention across conversation turns
+import { parseLLMResponse } from "./parse-llm-response.js";
+function extractConversationHistory(metadata) {
+    if (!metadata)
+        return undefined;
+    const raw = metadata["conversationHistory"];
+    if (!Array.isArray(raw))
+        return undefined;
+    const turns = [];
+    for (const entry of raw) {
+        if (typeof entry === "object" &&
+            entry !== null &&
+            typeof entry["role"] === "string" &&
+            typeof entry["content"] === "string") {
+            turns.push(entry);
+        }
+    }
+    return turns.length >= 2 ? turns : undefined;
+}
+export class MultiTurnConsistencyScorer {
+    llm;
+    name = "multi-turn-consistency";
+    constructor(llm) {
+        this.llm = llm;
+    }
+    async score(input) {
+        const history = extractConversationHistory(input.metadata);
+        if (!history) {
+            return { name: this.name, score: 0, reasoning: "No conversation history in metadata (need >= 2 turns)" };
+        }
+        const transcript = history.map((t) => `[${t.role}]: ${t.content}`).join("\n");
+        const prompt = `Evaluate context retention across this multi-turn conversation. Did the assistant maintain awareness of previously stated facts, requests, and context throughout the conversation?
+
+Conversation:
+${transcript}
+
+Final output: ${input.output}
+
+Evaluate:
+1. Does the assistant contradict earlier statements?
+2. Does it forget previously provided information?
+3. Does it ask questions already answered?
+4. Does it maintain a coherent understanding of the user's evolving needs?
+
+Score 1.0 for perfect context retention. Score 0.0 for complete context loss.
+
+Respond EXACTLY in this format:
+SCORE: <number from 0.0 to 1.0>
+REASONING: <one sentence explanation>`;
+        const response = await this.llm.evaluate(prompt);
+        const { score, reasoning } = parseLLMResponse(response, this.name);
+        return { name: this.name, score, reasoning };
+    }
+}
+//# sourceMappingURL=multi-turn-consistency-scorer.js.map

package/dist/eval/scorers/multi-turn-consistency-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi-turn-consistency-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/multi-turn-consistency-scorer.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAG3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAO3D,SAAS,0BAA0B,CAAC,QAA6C;IAC/E,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,qBAAqB,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,KAAK,GAAuB,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QACxB,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,OAAQ,KAAiC,CAAC,MAAM,CAAC,KAAK,QAAQ;YAC9D,OAAQ,KAAiC,CAAC,SAAS,CAAC,KAAK,QAAQ,EACjE,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,KAAyB,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED,MAAM,OAAO,0BAA0B;IAGR;IAFpB,IAAI,GAAG,wBAAwB,CAAC;IAEzC,YAA6B,GAAc;QAAd,QAAG,GAAH,GAAG,CAAW;IAAG,CAAC;IAE/C,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,OAAO,GAAG,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,uDAAuD,EAAE,CAAC;QAC3G,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9E,MAAM,MAAM,GAAG;;;EAGjB,UAAU;;gBAEI,KAAK,CAAC,MAAM;;;;;;;;;;;;sCAYU,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACnE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC/C,CAAC;CACF"}

package/dist/eval/scorers/policy-adherence-scorer.d.ts

@@ -0,0 +1,9 @@
+import type { EvalInput, EvalScore, Scorer, ScorerLLM } from "../types.js";
+export declare class PolicyAdherenceScorer implements Scorer {
+    private readonly llm;
+    private readonly policies;
+    readonly name = "policy-adherence";
+    constructor(llm: ScorerLLM, policies: readonly string[]);
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=policy-adherence-scorer.d.ts.map

package/dist/eval/scorers/policy-adherence-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy-adherence-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/policy-adherence-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAG3E,qBAAa,qBAAsB,YAAW,MAAM;IAIhD,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ3B,QAAQ,CAAC,IAAI,sBAAsB;gBAGhB,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,SAAS,MAAM,EAAE;IAGxC,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAwBlD"}

package/dist/eval/scorers/policy-adherence-scorer.js

@@ -0,0 +1,34 @@
+// PolicyAdherenceScorer: evaluates whether output follows business policy rules
+import { parseLLMResponse } from "./parse-llm-response.js";
+export class PolicyAdherenceScorer {
+    llm;
+    policies;
+    name = "policy-adherence";
+    constructor(llm, policies) {
+        this.llm = llm;
+        this.policies = policies;
+    }
+    async score(input) {
+        if (this.policies.length === 0) {
+            return { name: this.name, score: 0, reasoning: "No policies configured" };
+        }
+        const policyList = this.policies.map((p, i) => `${i + 1}. ${p}`).join("\n");
+        const prompt = `Evaluate policy adherence. Does the output comply with ALL of the following business policies?
+
+Policies:
+${policyList}
+
+Input: ${input.input}
+Output: ${input.output}
+
+Score 1.0 if the output fully adheres to all policies. Score 0.0 if it violates any policy. Use intermediate scores for partial adherence.
+
+Respond EXACTLY in this format:
+SCORE: <number from 0.0 to 1.0>
+REASONING: <one sentence explanation>`;
+        const response = await this.llm.evaluate(prompt);
+        const { score, reasoning } = parseLLMResponse(response, this.name);
+        return { name: this.name, score, reasoning };
+    }
+}
+//# sourceMappingURL=policy-adherence-scorer.js.map

package/dist/eval/scorers/policy-adherence-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy-adherence-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/policy-adherence-scorer.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAGhF,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,MAAM,OAAO,qBAAqB;IAIb;IACA;IAJV,IAAI,GAAG,kBAAkB,CAAC;IAEnC,YACmB,GAAc,EACd,QAA2B;QAD3B,QAAG,GAAH,GAAG,CAAW;QACd,aAAQ,GAAR,QAAQ,CAAmB;IAC3C,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,wBAAwB,EAAE,CAAC;QAC5E,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG;;;EAGjB,UAAU;;SAEH,KAAK,CAAC,KAAK;UACV,KAAK,CAAC,MAAM;;;;;;sCAMgB,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACnE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC/C,CAAC;CACF"}

package/dist/eval/scorers/resolution-scorer.d.ts

@@ -0,0 +1,6 @@
+import type { EvalInput, EvalScore, Scorer } from "../types.js";
+export declare class ResolutionScorer implements Scorer {
+    readonly name = "resolution";
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=resolution-scorer.d.ts.map

package/dist/eval/scorers/resolution-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolution-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/resolution-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAShE,qBAAa,gBAAiB,YAAW,MAAM;IAC7C,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAclD"}

package/dist/eval/scorers/resolution-scorer.js

@@ -0,0 +1,25 @@
+// ResolutionScorer: rule-based scorer mapping resolution status to eval score
+const RESOLUTION_SCORES = {
+    resolved: 1.0,
+    partial: 0.5,
+    ambiguous: 0.25,
+    unresolved: 0.0,
+};
+export class ResolutionScorer {
+    name = "resolution";
+    async score(input) {
+        const resolution = input.metadata?.resolution;
+        if (!resolution?.status) {
+            return { name: this.name, score: 0, reasoning: "No resolution data in metadata" };
+        }
+        const score = RESOLUTION_SCORES[resolution.status] ?? 0;
+        const confidence = resolution.confidence ?? 1.0;
+        const weighted = score * confidence;
+        return {
+            name: this.name,
+            score: weighted,
+            reasoning: `Resolution: ${resolution.status} (confidence: ${confidence})`,
+        };
+    }
+}
+//# sourceMappingURL=resolution-scorer.js.map

package/dist/eval/scorers/resolution-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolution-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/resolution-scorer.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAI9E,MAAM,iBAAiB,GAA2B;IAChD,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;IACZ,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,GAAG;CAChB,CAAC;AAEF,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,YAAY,CAAC;IAE7B,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAkE,CAAC;QACtG,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,gCAAgC,EAAE,CAAC;QACpF,CAAC;QACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC;QAChD,MAAM,QAAQ,GAAG,KAAK,GAAG,UAAU,CAAC;QACpC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,QAAQ;YACf,SAAS,EAAE,eAAe,UAAU,CAAC,MAAM,iBAAiB,UAAU,GAAG;SAC1E,CAAC;IACJ,CAAC;CACF"}

package/dist/eval/scorers/routing-accuracy-scorer.d.ts

@@ -0,0 +1,6 @@
+import type { EvalInput, EvalScore, Scorer } from "../types.js";
+export declare class RoutingAccuracyScorer implements Scorer {
+    readonly name = "routing-accuracy";
+    score(input: EvalInput): Promise<EvalScore>;
+}
+//# sourceMappingURL=routing-accuracy-scorer.d.ts.map

package/dist/eval/scorers/routing-accuracy-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"routing-accuracy-scorer.d.ts","sourceRoot":"","sources":["../../../src/eval/scorers/routing-accuracy-scorer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAEhE,qBAAa,qBAAsB,YAAW,MAAM;IAClD,QAAQ,CAAC,IAAI,sBAAsB;IAE7B,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;CAiBlD"}

package/dist/eval/scorers/routing-accuracy-scorer.js

@@ -0,0 +1,19 @@
+// RoutingAccuracyScorer: rule-based scorer checking if the correct agent handled the message
+export class RoutingAccuracyScorer {
+    name = "routing-accuracy";
+    async score(input) {
+        const expected = input.metadata?.expectedAgentId;
+        if (!expected) {
+            return { name: this.name, score: 0, reasoning: "No expectedAgentId in metadata" };
+        }
+        const actual = input.metadata?.activeAgentId;
+        if (!actual) {
+            return { name: this.name, score: 0, reasoning: "No activeAgentId in metadata" };
+        }
+        if (actual === expected) {
+            return { name: this.name, score: 1, reasoning: `Correct: routed to "${actual}"` };
+        }
+        return { name: this.name, score: 0, reasoning: `Expected "${expected}", got "${actual}"` };
+    }
+}
+//# sourceMappingURL=routing-accuracy-scorer.js.map

package/dist/eval/scorers/routing-accuracy-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"routing-accuracy-scorer.js","sourceRoot":"","sources":["../../../src/eval/scorers/routing-accuracy-scorer.ts"],"names":[],"mappings":"AAAA,6FAA6F;AAI7F,MAAM,OAAO,qBAAqB;IACvB,IAAI,GAAG,kBAAkB,CAAC;IAEnC,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,eAAqC,CAAC;QACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,gCAAgC,EAAE,CAAC;QACpF,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,EAAE,aAAmC,CAAC;QACnE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,8BAA8B,EAAE,CAAC;QAClF,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,uBAAuB,MAAM,GAAG,EAAE,CAAC;QACpF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,aAAa,QAAQ,WAAW,MAAM,GAAG,EAAE,CAAC;IAC7F,CAAC;CACF"}