@juspay/neurolink 9.36.1 → 9.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/auth/errors.d.ts +1 -1
  3. package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
  4. package/dist/auth/providers/BaseAuthProvider.d.ts +1 -1
  5. package/dist/browser/neurolink.min.js +921 -423
  6. package/dist/cli/commands/evaluate.d.ts +48 -0
  7. package/dist/cli/commands/evaluate.js +955 -0
  8. package/dist/cli/parser.js +4 -1
  9. package/dist/evaluation/BatchEvaluator.d.ts +163 -0
  10. package/dist/evaluation/BatchEvaluator.js +267 -0
  11. package/dist/evaluation/EvaluationAggregator.d.ts +272 -0
  12. package/dist/evaluation/EvaluationAggregator.js +377 -0
  13. package/dist/evaluation/EvaluatorFactory.d.ts +113 -0
  14. package/dist/evaluation/EvaluatorFactory.js +280 -0
  15. package/dist/evaluation/EvaluatorRegistry.d.ts +160 -0
  16. package/dist/evaluation/EvaluatorRegistry.js +184 -0
  17. package/dist/evaluation/errors/EvaluationError.d.ts +189 -0
  18. package/dist/evaluation/errors/EvaluationError.js +206 -0
  19. package/dist/evaluation/errors/index.d.ts +4 -0
  20. package/dist/evaluation/errors/index.js +4 -0
  21. package/dist/evaluation/hooks/index.d.ts +6 -0
  22. package/dist/evaluation/hooks/index.js +6 -0
  23. package/dist/evaluation/hooks/langfuseAdapter.d.ts +99 -0
  24. package/dist/evaluation/hooks/langfuseAdapter.js +172 -0
  25. package/dist/evaluation/hooks/observabilityHooks.d.ts +129 -0
  26. package/dist/evaluation/hooks/observabilityHooks.js +181 -0
  27. package/dist/evaluation/index.d.ts +11 -2
  28. package/dist/evaluation/index.js +15 -0
  29. package/dist/evaluation/pipeline/evaluationPipeline.d.ts +114 -0
  30. package/dist/evaluation/pipeline/evaluationPipeline.js +381 -0
  31. package/dist/evaluation/pipeline/index.d.ts +8 -0
  32. package/dist/evaluation/pipeline/index.js +8 -0
  33. package/dist/evaluation/pipeline/pipelineBuilder.d.ts +126 -0
  34. package/dist/evaluation/pipeline/pipelineBuilder.js +260 -0
  35. package/dist/evaluation/pipeline/presets.d.ts +66 -0
  36. package/dist/evaluation/pipeline/presets.js +224 -0
  37. package/dist/evaluation/pipeline/strategies/batchStrategy.d.ts +99 -0
  38. package/dist/evaluation/pipeline/strategies/batchStrategy.js +238 -0
  39. package/dist/evaluation/pipeline/strategies/index.d.ts +6 -0
  40. package/dist/evaluation/pipeline/strategies/index.js +6 -0
  41. package/dist/evaluation/pipeline/strategies/samplingStrategy.d.ts +76 -0
  42. package/dist/evaluation/pipeline/strategies/samplingStrategy.js +238 -0
  43. package/dist/evaluation/reporting/index.d.ts +6 -0
  44. package/dist/evaluation/reporting/index.js +6 -0
  45. package/dist/evaluation/reporting/metricsCollector.d.ts +147 -0
  46. package/dist/evaluation/reporting/metricsCollector.js +285 -0
  47. package/dist/evaluation/reporting/reportGenerator.d.ts +90 -0
  48. package/dist/evaluation/reporting/reportGenerator.js +374 -0
  49. package/dist/evaluation/scorers/baseScorer.d.ts +83 -0
  50. package/dist/evaluation/scorers/baseScorer.js +232 -0
  51. package/dist/evaluation/scorers/customScorerUtils.d.ts +95 -0
  52. package/dist/evaluation/scorers/customScorerUtils.js +381 -0
  53. package/dist/evaluation/scorers/index.d.ts +10 -0
  54. package/dist/evaluation/scorers/index.js +16 -0
  55. package/dist/evaluation/scorers/llm/answerRelevancyScorer.d.ts +12 -0
  56. package/dist/evaluation/scorers/llm/answerRelevancyScorer.js +99 -0
  57. package/dist/evaluation/scorers/llm/baseLLMScorer.d.ts +71 -0
  58. package/dist/evaluation/scorers/llm/baseLLMScorer.js +281 -0
  59. package/dist/evaluation/scorers/llm/biasDetectionScorer.d.ts +12 -0
  60. package/dist/evaluation/scorers/llm/biasDetectionScorer.js +127 -0
  61. package/dist/evaluation/scorers/llm/contextPrecisionScorer.d.ts +12 -0
  62. package/dist/evaluation/scorers/llm/contextPrecisionScorer.js +92 -0
  63. package/dist/evaluation/scorers/llm/contextRelevancyScorer.d.ts +12 -0
  64. package/dist/evaluation/scorers/llm/contextRelevancyScorer.js +107 -0
  65. package/dist/evaluation/scorers/llm/faithfulnessScorer.d.ts +12 -0
  66. package/dist/evaluation/scorers/llm/faithfulnessScorer.js +121 -0
  67. package/dist/evaluation/scorers/llm/hallucinationScorer.d.ts +12 -0
  68. package/dist/evaluation/scorers/llm/hallucinationScorer.js +140 -0
  69. package/dist/evaluation/scorers/llm/index.d.ts +15 -0
  70. package/dist/evaluation/scorers/llm/index.js +16 -0
  71. package/dist/evaluation/scorers/llm/promptAlignmentScorer.d.ts +12 -0
  72. package/dist/evaluation/scorers/llm/promptAlignmentScorer.js +106 -0
  73. package/dist/evaluation/scorers/llm/summarizationScorer.d.ts +12 -0
  74. package/dist/evaluation/scorers/llm/summarizationScorer.js +114 -0
  75. package/dist/evaluation/scorers/llm/toneConsistencyScorer.d.ts +12 -0
  76. package/dist/evaluation/scorers/llm/toneConsistencyScorer.js +106 -0
  77. package/dist/evaluation/scorers/llm/toxicityScorer.d.ts +12 -0
  78. package/dist/evaluation/scorers/llm/toxicityScorer.js +121 -0
  79. package/dist/evaluation/scorers/rule/baseRuleScorer.d.ts +77 -0
  80. package/dist/evaluation/scorers/rule/baseRuleScorer.js +233 -0
  81. package/dist/evaluation/scorers/rule/contentSimilarityScorer.d.ts +108 -0
  82. package/dist/evaluation/scorers/rule/contentSimilarityScorer.js +350 -0
  83. package/dist/evaluation/scorers/rule/formatScorer.d.ts +147 -0
  84. package/dist/evaluation/scorers/rule/formatScorer.js +470 -0
  85. package/dist/evaluation/scorers/rule/index.d.ts +9 -0
  86. package/dist/evaluation/scorers/rule/index.js +10 -0
  87. package/dist/evaluation/scorers/rule/keywordCoverageScorer.d.ts +83 -0
  88. package/dist/evaluation/scorers/rule/keywordCoverageScorer.js +347 -0
  89. package/dist/evaluation/scorers/rule/lengthScorer.d.ts +105 -0
  90. package/dist/evaluation/scorers/rule/lengthScorer.js +351 -0
  91. package/dist/evaluation/scorers/scorerBuilder.d.ts +161 -0
  92. package/dist/evaluation/scorers/scorerBuilder.js +420 -0
  93. package/dist/evaluation/scorers/scorerRegistry.d.ts +62 -0
  94. package/dist/evaluation/scorers/scorerRegistry.js +467 -0
  95. package/dist/index.d.ts +37 -25
  96. package/dist/index.js +65 -26
  97. package/dist/lib/auth/providers/BaseAuthProvider.d.ts +1 -1
  98. package/dist/lib/evaluation/BatchEvaluator.d.ts +163 -0
  99. package/dist/lib/evaluation/BatchEvaluator.js +268 -0
  100. package/dist/lib/evaluation/EvaluationAggregator.d.ts +272 -0
  101. package/dist/lib/evaluation/EvaluationAggregator.js +378 -0
  102. package/dist/lib/evaluation/EvaluatorFactory.d.ts +113 -0
  103. package/dist/lib/evaluation/EvaluatorFactory.js +281 -0
  104. package/dist/lib/evaluation/EvaluatorRegistry.d.ts +160 -0
  105. package/dist/lib/evaluation/EvaluatorRegistry.js +185 -0
  106. package/dist/lib/evaluation/errors/EvaluationError.d.ts +189 -0
  107. package/dist/lib/evaluation/errors/EvaluationError.js +207 -0
  108. package/dist/lib/evaluation/errors/index.d.ts +4 -0
  109. package/dist/lib/evaluation/errors/index.js +5 -0
  110. package/dist/lib/evaluation/hooks/index.d.ts +6 -0
  111. package/dist/lib/evaluation/hooks/index.js +7 -0
  112. package/dist/lib/evaluation/hooks/langfuseAdapter.d.ts +99 -0
  113. package/dist/lib/evaluation/hooks/langfuseAdapter.js +173 -0
  114. package/dist/lib/evaluation/hooks/observabilityHooks.d.ts +129 -0
  115. package/dist/lib/evaluation/hooks/observabilityHooks.js +182 -0
  116. package/dist/lib/evaluation/index.d.ts +11 -2
  117. package/dist/lib/evaluation/index.js +15 -0
  118. package/dist/lib/evaluation/pipeline/evaluationPipeline.d.ts +114 -0
  119. package/dist/lib/evaluation/pipeline/evaluationPipeline.js +382 -0
  120. package/dist/lib/evaluation/pipeline/index.d.ts +8 -0
  121. package/dist/lib/evaluation/pipeline/index.js +9 -0
  122. package/dist/lib/evaluation/pipeline/pipelineBuilder.d.ts +126 -0
  123. package/dist/lib/evaluation/pipeline/pipelineBuilder.js +261 -0
  124. package/dist/lib/evaluation/pipeline/presets.d.ts +66 -0
  125. package/dist/lib/evaluation/pipeline/presets.js +225 -0
  126. package/dist/lib/evaluation/pipeline/strategies/batchStrategy.d.ts +99 -0
  127. package/dist/lib/evaluation/pipeline/strategies/batchStrategy.js +239 -0
  128. package/dist/lib/evaluation/pipeline/strategies/index.d.ts +6 -0
  129. package/dist/lib/evaluation/pipeline/strategies/index.js +7 -0
  130. package/dist/lib/evaluation/pipeline/strategies/samplingStrategy.d.ts +76 -0
  131. package/dist/lib/evaluation/pipeline/strategies/samplingStrategy.js +239 -0
  132. package/dist/lib/evaluation/reporting/index.d.ts +6 -0
  133. package/dist/lib/evaluation/reporting/index.js +7 -0
  134. package/dist/lib/evaluation/reporting/metricsCollector.d.ts +147 -0
  135. package/dist/lib/evaluation/reporting/metricsCollector.js +286 -0
  136. package/dist/lib/evaluation/reporting/reportGenerator.d.ts +90 -0
  137. package/dist/lib/evaluation/reporting/reportGenerator.js +375 -0
  138. package/dist/lib/evaluation/scorers/baseScorer.d.ts +83 -0
  139. package/dist/lib/evaluation/scorers/baseScorer.js +233 -0
  140. package/dist/lib/evaluation/scorers/customScorerUtils.d.ts +95 -0
  141. package/dist/lib/evaluation/scorers/customScorerUtils.js +382 -0
  142. package/dist/lib/evaluation/scorers/index.d.ts +10 -0
  143. package/dist/lib/evaluation/scorers/index.js +17 -0
  144. package/dist/lib/evaluation/scorers/llm/answerRelevancyScorer.d.ts +12 -0
  145. package/dist/lib/evaluation/scorers/llm/answerRelevancyScorer.js +100 -0
  146. package/dist/lib/evaluation/scorers/llm/baseLLMScorer.d.ts +71 -0
  147. package/dist/lib/evaluation/scorers/llm/baseLLMScorer.js +282 -0
  148. package/dist/lib/evaluation/scorers/llm/biasDetectionScorer.d.ts +12 -0
  149. package/dist/lib/evaluation/scorers/llm/biasDetectionScorer.js +128 -0
  150. package/dist/lib/evaluation/scorers/llm/contextPrecisionScorer.d.ts +12 -0
  151. package/dist/lib/evaluation/scorers/llm/contextPrecisionScorer.js +93 -0
  152. package/dist/lib/evaluation/scorers/llm/contextRelevancyScorer.d.ts +12 -0
  153. package/dist/lib/evaluation/scorers/llm/contextRelevancyScorer.js +108 -0
  154. package/dist/lib/evaluation/scorers/llm/faithfulnessScorer.d.ts +12 -0
  155. package/dist/lib/evaluation/scorers/llm/faithfulnessScorer.js +122 -0
  156. package/dist/lib/evaluation/scorers/llm/hallucinationScorer.d.ts +12 -0
  157. package/dist/lib/evaluation/scorers/llm/hallucinationScorer.js +141 -0
  158. package/dist/lib/evaluation/scorers/llm/index.d.ts +15 -0
  159. package/dist/lib/evaluation/scorers/llm/index.js +17 -0
  160. package/dist/lib/evaluation/scorers/llm/promptAlignmentScorer.d.ts +12 -0
  161. package/dist/lib/evaluation/scorers/llm/promptAlignmentScorer.js +107 -0
  162. package/dist/lib/evaluation/scorers/llm/summarizationScorer.d.ts +12 -0
  163. package/dist/lib/evaluation/scorers/llm/summarizationScorer.js +115 -0
  164. package/dist/lib/evaluation/scorers/llm/toneConsistencyScorer.d.ts +12 -0
  165. package/dist/lib/evaluation/scorers/llm/toneConsistencyScorer.js +107 -0
  166. package/dist/lib/evaluation/scorers/llm/toxicityScorer.d.ts +12 -0
  167. package/dist/lib/evaluation/scorers/llm/toxicityScorer.js +122 -0
  168. package/dist/lib/evaluation/scorers/rule/baseRuleScorer.d.ts +77 -0
  169. package/dist/lib/evaluation/scorers/rule/baseRuleScorer.js +234 -0
  170. package/dist/lib/evaluation/scorers/rule/contentSimilarityScorer.d.ts +108 -0
  171. package/dist/lib/evaluation/scorers/rule/contentSimilarityScorer.js +351 -0
  172. package/dist/lib/evaluation/scorers/rule/formatScorer.d.ts +147 -0
  173. package/dist/lib/evaluation/scorers/rule/formatScorer.js +471 -0
  174. package/dist/lib/evaluation/scorers/rule/index.d.ts +9 -0
  175. package/dist/lib/evaluation/scorers/rule/index.js +11 -0
  176. package/dist/lib/evaluation/scorers/rule/keywordCoverageScorer.d.ts +83 -0
  177. package/dist/lib/evaluation/scorers/rule/keywordCoverageScorer.js +348 -0
  178. package/dist/lib/evaluation/scorers/rule/lengthScorer.d.ts +105 -0
  179. package/dist/lib/evaluation/scorers/rule/lengthScorer.js +352 -0
  180. package/dist/lib/evaluation/scorers/scorerBuilder.d.ts +161 -0
  181. package/dist/lib/evaluation/scorers/scorerBuilder.js +421 -0
  182. package/dist/lib/evaluation/scorers/scorerRegistry.d.ts +62 -0
  183. package/dist/lib/evaluation/scorers/scorerRegistry.js +468 -0
  184. package/dist/lib/index.d.ts +37 -25
  185. package/dist/lib/index.js +65 -26
  186. package/dist/lib/neurolink.d.ts +204 -0
  187. package/dist/lib/neurolink.js +296 -0
  188. package/dist/lib/types/index.d.ts +3 -1
  189. package/dist/lib/types/index.js +3 -2
  190. package/dist/lib/types/scorerTypes.d.ts +423 -0
  191. package/dist/lib/types/scorerTypes.js +6 -0
  192. package/dist/lib/utils/errorHandling.d.ts +20 -0
  193. package/dist/lib/utils/errorHandling.js +60 -0
  194. package/dist/neurolink.d.ts +204 -0
  195. package/dist/neurolink.js +296 -0
  196. package/dist/types/index.d.ts +3 -1
  197. package/dist/types/index.js +3 -2
  198. package/dist/types/scorerTypes.d.ts +423 -0
  199. package/dist/types/scorerTypes.js +5 -0
  200. package/dist/utils/errorHandling.d.ts +20 -0
  201. package/dist/utils/errorHandling.js +60 -0
  202. package/package.json +1 -1
@@ -0,0 +1,184 @@
1
+ /**
2
+ * @file EvaluatorRegistry - Registry for evaluation strategies.
3
+ * Extends BaseRegistry to provide dynamic strategy registration and lookup.
4
+ */
5
+ import { BaseRegistry } from "../core/infrastructure/index.js";
6
+ import { createStrategyNotFoundError } from "./errors/EvaluationError.js";
7
+ import { withTimeout, ErrorFactory } from "../utils/errorHandling.js";
8
+ /**
9
+ * Registry for evaluation strategies.
10
+ * Allows dynamic registration and retrieval of evaluation strategies.
11
+ *
12
+ * @example
13
+ * ```typescript
14
+ * // Register a custom strategy
15
+ * EvaluatorRegistry.getInstance().registerStrategy(
16
+ * 'custom-ragas',
17
+ * async () => ({
18
+ * evaluate: async (options, result) => { ... }
19
+ * }),
20
+ * {
21
+ * name: 'Custom RAGAS',
22
+ * description: 'Custom RAGAS implementation',
23
+ * requiresLLM: true,
24
+ * defaultModel: 'gpt-4',
25
+ * defaultProvider: 'openai',
26
+ * version: '1.0.0',
27
+ * features: ['custom-metrics']
28
+ * }
29
+ * );
30
+ *
31
+ * // Get a strategy
32
+ * const strategy = await EvaluatorRegistry.getInstance().getStrategy('ragas');
33
+ * ```
34
+ */
35
+ export class EvaluatorRegistry extends BaseRegistry {
36
+ static instance = null;
37
+ constructor() {
38
+ super();
39
+ }
40
+ /**
41
+ * Gets the singleton instance of the EvaluatorRegistry.
42
+ */
43
+ static getInstance() {
44
+ if (!EvaluatorRegistry.instance) {
45
+ EvaluatorRegistry.instance = new EvaluatorRegistry();
46
+ }
47
+ return EvaluatorRegistry.instance;
48
+ }
49
+ /**
50
+ * Resets the singleton instance (useful for testing).
51
+ */
52
+ static resetInstance() {
53
+ EvaluatorRegistry.instance = null;
54
+ }
55
+ /**
56
+ * Registers all built-in evaluation strategies.
57
+ * This is called automatically on first access.
58
+ */
59
+ async registerAll() {
60
+ // Register the built-in RAGAS strategy
61
+ this.registerStrategy("ragas", async () => {
62
+ const { RAGASEvaluator } = await import("./ragasEvaluator.js");
63
+ const { ContextBuilder } = await import("./contextBuilder.js");
64
+ return async (options, result, config) => {
65
+ const contextBuilder = new ContextBuilder();
66
+ const ragasEvaluator = new RAGASEvaluator(config?.evaluationModel, config?.provider, config?.threshold, config?.promptGenerator);
67
+ const evalContext = contextBuilder.buildContext(options, result);
68
+ const evaluationTimeoutMs = config?.options?.timeout ?? 60000;
69
+ const evaluationResult = await withTimeout(ragasEvaluator.evaluate(evalContext), evaluationTimeoutMs, ErrorFactory.evaluationTimeout("strategy evaluation", evaluationTimeoutMs));
70
+ return { evaluationResult, evalContext };
71
+ };
72
+ }, {
73
+ name: "RAGAS Evaluator",
74
+ description: "RAGAS-style LLM-as-judge evaluation with relevance, accuracy, and completeness metrics",
75
+ requiresLLM: true,
76
+ defaultModel: "gemini-1.5-flash",
77
+ defaultProvider: "vertex",
78
+ version: "1.0.0",
79
+ features: [
80
+ "relevance-scoring",
81
+ "accuracy-scoring",
82
+ "completeness-scoring",
83
+ "reasoning",
84
+ "suggestions",
85
+ ],
86
+ });
87
+ }
88
+ /**
89
+ * Registers an evaluation strategy with the registry.
90
+ *
91
+ * @param id - Unique identifier for the strategy
92
+ * @param factory - Factory function that creates the strategy
93
+ * @param metadata - Metadata about the strategy
94
+ */
95
+ registerStrategy(id, factory, metadata) {
96
+ this.register(id, factory, [], { metadata });
97
+ }
98
+ /**
99
+ * Gets an evaluation strategy by ID.
100
+ *
101
+ * @param id - The strategy identifier
102
+ * @returns The evaluation strategy function
103
+ * @throws {NeuroLinkFeatureError} If the strategy is not found
104
+ */
105
+ async getStrategy(id) {
106
+ const strategy = await this.get(id);
107
+ if (!strategy) {
108
+ const strategies = await this.listStrategies();
109
+ const available = strategies.map((s) => s.id);
110
+ throw createStrategyNotFoundError(id, available);
111
+ }
112
+ return strategy;
113
+ }
114
+ /**
115
+ * Checks if a strategy exists in the registry.
116
+ *
117
+ * @param id - The strategy identifier
118
+ * @returns true if the strategy exists
119
+ */
120
+ async hasStrategy(id) {
121
+ await this.ensureInitialized();
122
+ return this.has(id);
123
+ }
124
+ /**
125
+ * Lists all registered strategies with their metadata.
126
+ *
127
+ * @returns Array of strategy IDs and their metadata
128
+ */
129
+ async listStrategies() {
130
+ await this.ensureInitialized();
131
+ return this.list();
132
+ }
133
+ /**
134
+ * Gets the metadata for a specific strategy.
135
+ *
136
+ * @param id - The strategy identifier
137
+ * @returns The strategy metadata or undefined if not found
138
+ */
139
+ async getStrategyMetadata(id) {
140
+ await this.ensureInitialized();
141
+ const entry = this.items.get(id);
142
+ return entry?.metadata;
143
+ }
144
+ /**
145
+ * Unregisters a strategy from the registry.
146
+ *
147
+ * @param id - The strategy identifier
148
+ * @returns true if the strategy was removed, false if it didn't exist
149
+ */
150
+ async unregisterStrategy(id) {
151
+ await this.ensureInitialized();
152
+ if (this.items.has(id)) {
153
+ this.items.delete(id);
154
+ return true;
155
+ }
156
+ return false;
157
+ }
158
+ /**
159
+ * Gets strategies that support a specific feature.
160
+ *
161
+ * @param feature - The feature to filter by
162
+ * @returns Array of strategy IDs that support the feature
163
+ */
164
+ async getStrategiesWithFeature(feature) {
165
+ await this.ensureInitialized();
166
+ return this.list()
167
+ .filter((item) => item.metadata.features.includes(feature))
168
+ .map((item) => item.id);
169
+ }
170
+ /**
171
+ * Gets strategies that use a specific provider.
172
+ *
173
+ * @param provider - The provider to filter by
174
+ * @returns Array of strategy IDs that use the provider
175
+ */
176
+ async getStrategiesByProvider(provider) {
177
+ await this.ensureInitialized();
178
+ return this.list()
179
+ .filter((item) => item.metadata.defaultProvider === provider)
180
+ .map((item) => item.id);
181
+ }
182
+ }
183
+ // Export singleton instance getter for convenience
184
+ export const getEvaluatorRegistry = () => EvaluatorRegistry.getInstance();
@@ -0,0 +1,189 @@
1
+ /**
2
+ * @file Typed error hierarchy for the Evaluation/Scoring System.
3
+ * Uses NeuroLinkFeatureError and createErrorFactory from core infrastructure.
4
+ */
5
+ import { NeuroLinkFeatureError } from "../../core/infrastructure/index.js";
6
+ import type { EnhancedEvaluationContext } from "../../types/evaluationTypes.js";
7
+ /**
8
+ * Error codes for the Evaluation feature.
9
+ * These codes help identify specific error scenarios for proper handling.
10
+ */
11
+ export declare const EvaluationErrorCodes: {
12
+ /** The evaluation process itself failed */
13
+ readonly EVALUATION_FAILED: "EVALUATION_FAILED";
14
+ /** Failed to parse the evaluation response from the judge LLM */
15
+ readonly PARSE_ERROR: "PARSE_ERROR";
16
+ /** The requested evaluation strategy was not found in the registry */
17
+ readonly STRATEGY_NOT_FOUND: "STRATEGY_NOT_FOUND";
18
+ /** The AI provider for evaluation failed */
19
+ readonly PROVIDER_ERROR: "PROVIDER_ERROR";
20
+ /** Configuration for evaluation is invalid or missing */
21
+ readonly CONFIGURATION_ERROR: "CONFIGURATION_ERROR";
22
+ /** The custom evaluator function threw an error */
23
+ readonly CUSTOM_EVALUATOR_ERROR: "CUSTOM_EVALUATOR_ERROR";
24
+ /** Batch evaluation failed for one or more items */
25
+ readonly BATCH_EVALUATION_ERROR: "BATCH_EVALUATION_ERROR";
26
+ /** Aggregation of evaluation results failed */
27
+ readonly AGGREGATION_ERROR: "AGGREGATION_ERROR";
28
+ /** Registry operation failed */
29
+ readonly REGISTRY_ERROR: "REGISTRY_ERROR";
30
+ /** Maximum retry attempts exceeded */
31
+ readonly MAX_RETRIES_EXCEEDED: "MAX_RETRIES_EXCEEDED";
32
+ /** Timeout during evaluation */
33
+ readonly TIMEOUT_ERROR: "TIMEOUT_ERROR";
34
+ /** Rate limit hit during evaluation */
35
+ readonly RATE_LIMIT_ERROR: "RATE_LIMIT_ERROR";
36
+ };
37
+ /**
38
+ * Type for evaluation error codes
39
+ */
40
+ export type EvaluationErrorCode = (typeof EvaluationErrorCodes)[keyof typeof EvaluationErrorCodes];
41
+ /**
42
+ * Factory for creating typed evaluation errors.
43
+ * Uses the createErrorFactory pattern from core infrastructure.
44
+ */
45
+ export declare const evaluationErrors: {
46
+ codes: {
47
+ /** The evaluation process itself failed */
48
+ readonly EVALUATION_FAILED: "EVALUATION_FAILED";
49
+ /** Failed to parse the evaluation response from the judge LLM */
50
+ readonly PARSE_ERROR: "PARSE_ERROR";
51
+ /** The requested evaluation strategy was not found in the registry */
52
+ readonly STRATEGY_NOT_FOUND: "STRATEGY_NOT_FOUND";
53
+ /** The AI provider for evaluation failed */
54
+ readonly PROVIDER_ERROR: "PROVIDER_ERROR";
55
+ /** Configuration for evaluation is invalid or missing */
56
+ readonly CONFIGURATION_ERROR: "CONFIGURATION_ERROR";
57
+ /** The custom evaluator function threw an error */
58
+ readonly CUSTOM_EVALUATOR_ERROR: "CUSTOM_EVALUATOR_ERROR";
59
+ /** Batch evaluation failed for one or more items */
60
+ readonly BATCH_EVALUATION_ERROR: "BATCH_EVALUATION_ERROR";
61
+ /** Aggregation of evaluation results failed */
62
+ readonly AGGREGATION_ERROR: "AGGREGATION_ERROR";
63
+ /** Registry operation failed */
64
+ readonly REGISTRY_ERROR: "REGISTRY_ERROR";
65
+ /** Maximum retry attempts exceeded */
66
+ readonly MAX_RETRIES_EXCEEDED: "MAX_RETRIES_EXCEEDED";
67
+ /** Timeout during evaluation */
68
+ readonly TIMEOUT_ERROR: "TIMEOUT_ERROR";
69
+ /** Rate limit hit during evaluation */
70
+ readonly RATE_LIMIT_ERROR: "RATE_LIMIT_ERROR";
71
+ };
72
+ create: (code: "PROVIDER_ERROR" | "CONFIGURATION_ERROR" | "EVALUATION_FAILED" | "PARSE_ERROR" | "STRATEGY_NOT_FOUND" | "CUSTOM_EVALUATOR_ERROR" | "BATCH_EVALUATION_ERROR" | "AGGREGATION_ERROR" | "REGISTRY_ERROR" | "MAX_RETRIES_EXCEEDED" | "TIMEOUT_ERROR" | "RATE_LIMIT_ERROR", message: string, options?: {
73
+ retryable?: boolean;
74
+ details?: Record<string, unknown>;
75
+ cause?: Error;
76
+ } | undefined) => NeuroLinkFeatureError;
77
+ };
78
+ /**
79
+ * Extended evaluation context for error details.
80
+ * Provides rich debugging information when errors occur.
81
+ */
82
+ export interface EvaluationErrorContext {
83
+ /** Length of the user query (redacted for safety) */
84
+ userQueryLength?: number;
85
+ /** Length of the AI response (redacted for safety) */
86
+ aiResponseLength?: number;
87
+ /** The current attempt number */
88
+ attemptNumber?: number;
89
+ /** Previous evaluation scores if any */
90
+ previousScores?: number[];
91
+ /** The evaluation strategy in use */
92
+ strategy?: string;
93
+ /** The evaluation model being used */
94
+ evaluationModel?: string;
95
+ /** The provider being used */
96
+ provider?: string;
97
+ /** Length of the raw response (redacted for safety) */
98
+ rawResponseLength?: number;
99
+ /** Any additional context */
100
+ additionalContext?: Record<string, unknown>;
101
+ }
102
+ /**
103
+ * Checks if an error is retryable based on its code.
104
+ * Transient errors (timeout, rate limit, some provider errors) are retryable.
105
+ *
106
+ * @param error - The error to check
107
+ * @returns true if the error is retryable
108
+ */
109
+ export declare function isRetryableEvaluationError(error: NeuroLinkFeatureError): boolean;
110
+ /**
111
+ * Checks if an error is a NeuroLinkFeatureError from the Evaluation feature.
112
+ *
113
+ * @param error - The error to check
114
+ * @returns true if the error is an evaluation error
115
+ */
116
+ export declare function isEvaluationError(error: unknown): error is NeuroLinkFeatureError;
117
+ /**
118
+ * Helper function to create an evaluation failed error with context.
119
+ *
120
+ * @param message - The error message
121
+ * @param context - The evaluation context
122
+ * @param cause - The underlying cause error
123
+ * @returns A typed NeuroLinkFeatureError
124
+ */
125
+ export declare function createEvaluationFailedError(message: string, context?: EvaluationErrorContext, cause?: Error): NeuroLinkFeatureError;
126
+ /**
127
+ * Helper function to create a parse error with raw response.
128
+ *
129
+ * @param rawResponse - The raw response that failed to parse
130
+ * @param cause - The underlying parse error
131
+ * @returns A typed NeuroLinkFeatureError
132
+ */
133
+ export declare function createParseError(rawResponse: string, cause?: Error): NeuroLinkFeatureError;
134
+ /**
135
+ * Helper function to create a strategy not found error.
136
+ *
137
+ * @param strategyName - The name of the strategy that was not found
138
+ * @param availableStrategies - List of available strategies
139
+ * @returns A typed NeuroLinkFeatureError
140
+ */
141
+ export declare function createStrategyNotFoundError(strategyName: string, availableStrategies?: string[]): NeuroLinkFeatureError;
142
+ /**
143
+ * Helper function to create a provider error.
144
+ *
145
+ * @param message - The error message
146
+ * @param provider - The provider that failed
147
+ * @param cause - The underlying cause error
148
+ * @returns A typed NeuroLinkFeatureError
149
+ */
150
+ export declare function createProviderError(message: string, provider: string, cause?: Error, options?: {
151
+ retryable?: boolean;
152
+ }): NeuroLinkFeatureError;
153
+ /**
154
+ * Helper function to create a max retries exceeded error.
155
+ *
156
+ * @param attempts - The number of attempts made
157
+ * @param lastScore - The last evaluation score
158
+ * @param threshold - The passing threshold
159
+ * @param context - The evaluation context
160
+ * @returns A typed NeuroLinkFeatureError
161
+ */
162
+ export declare function createMaxRetriesExceededError(attempts: number, lastScore: number, threshold: number, context?: EvaluationErrorContext): NeuroLinkFeatureError;
163
+ /**
164
+ * Helper function to create a batch evaluation error.
165
+ *
166
+ * @param failedCount - Number of evaluations that failed
167
+ * @param totalCount - Total number of evaluations attempted
168
+ * @param errors - Array of individual errors
169
+ * @returns A typed NeuroLinkFeatureError
170
+ */
171
+ export declare function createBatchEvaluationError(failedCount: number, totalCount: number, errors: Array<{
172
+ index: number;
173
+ error: Error;
174
+ }>): NeuroLinkFeatureError;
175
+ /**
176
+ * Helper function to create a configuration error.
177
+ *
178
+ * @param message - The error message
179
+ * @param configIssue - Description of the configuration issue
180
+ * @returns A typed NeuroLinkFeatureError
181
+ */
182
+ export declare function createConfigurationError(message: string, configIssue: string): NeuroLinkFeatureError;
183
+ /**
184
+ * Converts an evaluation context to error context for debugging.
185
+ *
186
+ * @param context - The enhanced evaluation context
187
+ * @returns An EvaluationErrorContext
188
+ */
189
+ export declare function contextToErrorContext(context: EnhancedEvaluationContext): EvaluationErrorContext;
@@ -0,0 +1,206 @@
1
+ /**
2
+ * @file Typed error hierarchy for the Evaluation/Scoring System.
3
+ * Uses NeuroLinkFeatureError and createErrorFactory from core infrastructure.
4
+ */
5
+ import { NeuroLinkFeatureError, createErrorFactory, } from "../../core/infrastructure/index.js";
6
+ /**
7
+ * Error codes for the Evaluation feature.
8
+ * These codes help identify specific error scenarios for proper handling.
9
+ */
10
+ export const EvaluationErrorCodes = {
11
+ /** The evaluation process itself failed */
12
+ EVALUATION_FAILED: "EVALUATION_FAILED",
13
+ /** Failed to parse the evaluation response from the judge LLM */
14
+ PARSE_ERROR: "PARSE_ERROR",
15
+ /** The requested evaluation strategy was not found in the registry */
16
+ STRATEGY_NOT_FOUND: "STRATEGY_NOT_FOUND",
17
+ /** The AI provider for evaluation failed */
18
+ PROVIDER_ERROR: "PROVIDER_ERROR",
19
+ /** Configuration for evaluation is invalid or missing */
20
+ CONFIGURATION_ERROR: "CONFIGURATION_ERROR",
21
+ /** The custom evaluator function threw an error */
22
+ CUSTOM_EVALUATOR_ERROR: "CUSTOM_EVALUATOR_ERROR",
23
+ /** Batch evaluation failed for one or more items */
24
+ BATCH_EVALUATION_ERROR: "BATCH_EVALUATION_ERROR",
25
+ /** Aggregation of evaluation results failed */
26
+ AGGREGATION_ERROR: "AGGREGATION_ERROR",
27
+ /** Registry operation failed */
28
+ REGISTRY_ERROR: "REGISTRY_ERROR",
29
+ /** Maximum retry attempts exceeded */
30
+ MAX_RETRIES_EXCEEDED: "MAX_RETRIES_EXCEEDED",
31
+ /** Timeout during evaluation */
32
+ TIMEOUT_ERROR: "TIMEOUT_ERROR",
33
+ /** Rate limit hit during evaluation */
34
+ RATE_LIMIT_ERROR: "RATE_LIMIT_ERROR",
35
+ };
36
+ /**
37
+ * Factory for creating typed evaluation errors.
38
+ * Uses the createErrorFactory pattern from core infrastructure.
39
+ */
40
+ export const evaluationErrors = createErrorFactory("Evaluation", EvaluationErrorCodes);
41
+ /**
42
+ * Checks if an error is retryable based on its code.
43
+ * Transient errors (timeout, rate limit, some provider errors) are retryable.
44
+ *
45
+ * @param error - The error to check
46
+ * @returns true if the error is retryable
47
+ */
48
+ export function isRetryableEvaluationError(error) {
49
+ if (error.retryable) {
50
+ return true;
51
+ }
52
+ const retryableCodes = [
53
+ EvaluationErrorCodes.TIMEOUT_ERROR,
54
+ EvaluationErrorCodes.RATE_LIMIT_ERROR,
55
+ ];
56
+ return retryableCodes.includes(error.code);
57
+ }
58
+ /**
59
+ * Checks if an error is a NeuroLinkFeatureError from the Evaluation feature.
60
+ *
61
+ * @param error - The error to check
62
+ * @returns true if the error is an evaluation error
63
+ */
64
+ export function isEvaluationError(error) {
65
+ return (error instanceof NeuroLinkFeatureError && error.feature === "Evaluation");
66
+ }
67
+ /**
68
+ * Helper function to create an evaluation failed error with context.
69
+ *
70
+ * @param message - The error message
71
+ * @param context - The evaluation context
72
+ * @param cause - The underlying cause error
73
+ * @returns A typed NeuroLinkFeatureError
74
+ */
75
+ export function createEvaluationFailedError(message, context, cause) {
76
+ return evaluationErrors.create("EVALUATION_FAILED", message, {
77
+ retryable: false,
78
+ details: {
79
+ evaluationContext: context,
80
+ },
81
+ cause,
82
+ });
83
+ }
84
+ /**
85
+ * Helper function to create a parse error with raw response.
86
+ *
87
+ * @param rawResponse - The raw response that failed to parse
88
+ * @param cause - The underlying parse error
89
+ * @returns A typed NeuroLinkFeatureError
90
+ */
91
+ export function createParseError(rawResponse, cause) {
92
+ return evaluationErrors.create("PARSE_ERROR", "Failed to parse evaluation response from judge LLM", {
93
+ retryable: false,
94
+ details: {
95
+ rawResponseLength: rawResponse.length,
96
+ },
97
+ cause,
98
+ });
99
+ }
100
+ /**
101
+ * Helper function to create a strategy not found error.
102
+ *
103
+ * @param strategyName - The name of the strategy that was not found
104
+ * @param availableStrategies - List of available strategies
105
+ * @returns A typed NeuroLinkFeatureError
106
+ */
107
+ export function createStrategyNotFoundError(strategyName, availableStrategies = []) {
108
+ return evaluationErrors.create("STRATEGY_NOT_FOUND", `Evaluation strategy "${strategyName}" not found in registry`, {
109
+ retryable: false,
110
+ details: {
111
+ requestedStrategy: strategyName,
112
+ availableStrategies,
113
+ },
114
+ });
115
+ }
116
+ /**
117
+ * Helper function to create a provider error.
118
+ *
119
+ * @param message - The error message
120
+ * @param provider - The provider that failed
121
+ * @param cause - The underlying cause error
122
+ * @returns A typed NeuroLinkFeatureError
123
+ */
124
+ export function createProviderError(message, provider, cause, options) {
125
+ return evaluationErrors.create("PROVIDER_ERROR", message, {
126
+ retryable: options?.retryable ?? false,
127
+ details: {
128
+ provider,
129
+ errorMessage: cause?.message,
130
+ },
131
+ cause,
132
+ });
133
+ }
134
+ /**
135
+ * Helper function to create a max retries exceeded error.
136
+ *
137
+ * @param attempts - The number of attempts made
138
+ * @param lastScore - The last evaluation score
139
+ * @param threshold - The passing threshold
140
+ * @param context - The evaluation context
141
+ * @returns A typed NeuroLinkFeatureError
142
+ */
143
+ export function createMaxRetriesExceededError(attempts, lastScore, threshold, context) {
144
+ return evaluationErrors.create("MAX_RETRIES_EXCEEDED", `Maximum retry attempts (${attempts}) exceeded. Last score: ${lastScore}, threshold: ${threshold}`, {
145
+ retryable: false,
146
+ details: {
147
+ attempts,
148
+ lastScore,
149
+ threshold,
150
+ evaluationContext: context,
151
+ },
152
+ });
153
+ }
154
+ /**
155
+ * Helper function to create a batch evaluation error.
156
+ *
157
+ * @param failedCount - Number of evaluations that failed
158
+ * @param totalCount - Total number of evaluations attempted
159
+ * @param errors - Array of individual errors
160
+ * @returns A typed NeuroLinkFeatureError
161
+ */
162
+ export function createBatchEvaluationError(failedCount, totalCount, errors) {
163
+ return evaluationErrors.create("BATCH_EVALUATION_ERROR", `Batch evaluation failed: ${failedCount} of ${totalCount} evaluations failed`, {
164
+ retryable: false,
165
+ details: {
166
+ failedCount,
167
+ totalCount,
168
+ successCount: totalCount - failedCount,
169
+ errors: errors.map((e) => ({
170
+ index: e.index,
171
+ message: e.error.message,
172
+ })),
173
+ },
174
+ });
175
+ }
176
+ /**
177
+ * Helper function to create a configuration error.
178
+ *
179
+ * @param message - The error message
180
+ * @param configIssue - Description of the configuration issue
181
+ * @returns A typed NeuroLinkFeatureError
182
+ */
183
+ export function createConfigurationError(message, configIssue) {
184
+ return evaluationErrors.create("CONFIGURATION_ERROR", message, {
185
+ retryable: false,
186
+ details: {
187
+ configIssue,
188
+ },
189
+ });
190
+ }
191
+ /**
192
+ * Converts an evaluation context to error context for debugging.
193
+ *
194
+ * @param context - The enhanced evaluation context
195
+ * @returns An EvaluationErrorContext
196
+ */
197
+ export function contextToErrorContext(context) {
198
+ return {
199
+ userQueryLength: context.userQuery?.length,
200
+ aiResponseLength: context.aiResponse?.length,
201
+ attemptNumber: context.attemptNumber,
202
+ previousScores: context.previousEvaluations?.map((e) => e.finalScore),
203
+ evaluationModel: undefined, // Will be set by the evaluator
204
+ provider: context.provider,
205
+ };
206
+ }
@@ -0,0 +1,4 @@
1
+ /**
2
+ * @file Exports all evaluation error types and utilities.
3
+ */
4
+ export { EvaluationErrorCodes, type EvaluationErrorCode, evaluationErrors, type EvaluationErrorContext, isRetryableEvaluationError, isEvaluationError, createEvaluationFailedError, createParseError, createStrategyNotFoundError, createProviderError, createMaxRetriesExceededError, createBatchEvaluationError, createConfigurationError, contextToErrorContext, } from "./EvaluationError.js";
@@ -0,0 +1,4 @@
1
+ /**
2
+ * @file Exports all evaluation error types and utilities.
3
+ */
4
+ export { EvaluationErrorCodes, evaluationErrors, isRetryableEvaluationError, isEvaluationError, createEvaluationFailedError, createParseError, createStrategyNotFoundError, createProviderError, createMaxRetriesExceededError, createBatchEvaluationError, createConfigurationError, contextToErrorContext, } from "./EvaluationError.js";
@@ -0,0 +1,6 @@
1
+ /**
2
+ * @file Hooks Index
3
+ * Export all observability hooks
4
+ */
5
+ export { createLangfuseAdapter, createMockLangfuseClient, LangfuseAdapter, type LangfuseAdapterConfig, type LangfuseClient, startLangfuseAdapter, } from "./langfuseAdapter.js";
6
+ export { createConsoleLoggerHook, createMetricsCollectorHook, type EvaluationEvents, type EventHandler, ObservabilityHooks, observabilityHooks, pipelineToSpanAttributes, type SpanAttributes, scorerToSpanAttributes, } from "./observabilityHooks.js";
@@ -0,0 +1,6 @@
1
+ /**
2
+ * @file Hooks Index
3
+ * Export all observability hooks
4
+ */
5
+ export { createLangfuseAdapter, createMockLangfuseClient, LangfuseAdapter, startLangfuseAdapter, } from "./langfuseAdapter.js";
6
+ export { createConsoleLoggerHook, createMetricsCollectorHook, ObservabilityHooks, observabilityHooks, pipelineToSpanAttributes, scorerToSpanAttributes, } from "./observabilityHooks.js";