@juspay/neurolink 9.36.1 → 9.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/auth/errors.d.ts +1 -1
- package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
- package/dist/auth/providers/BaseAuthProvider.d.ts +1 -1
- package/dist/browser/neurolink.min.js +1105 -556
- package/dist/cli/commands/evaluate.d.ts +48 -0
- package/dist/cli/commands/evaluate.js +955 -0
- package/dist/cli/parser.js +4 -1
- package/dist/evaluation/BatchEvaluator.d.ts +163 -0
- package/dist/evaluation/BatchEvaluator.js +267 -0
- package/dist/evaluation/EvaluationAggregator.d.ts +272 -0
- package/dist/evaluation/EvaluationAggregator.js +377 -0
- package/dist/evaluation/EvaluatorFactory.d.ts +113 -0
- package/dist/evaluation/EvaluatorFactory.js +280 -0
- package/dist/evaluation/EvaluatorRegistry.d.ts +160 -0
- package/dist/evaluation/EvaluatorRegistry.js +184 -0
- package/dist/evaluation/errors/EvaluationError.d.ts +189 -0
- package/dist/evaluation/errors/EvaluationError.js +206 -0
- package/dist/evaluation/errors/index.d.ts +4 -0
- package/dist/evaluation/errors/index.js +4 -0
- package/dist/evaluation/hooks/index.d.ts +6 -0
- package/dist/evaluation/hooks/index.js +6 -0
- package/dist/evaluation/hooks/langfuseAdapter.d.ts +99 -0
- package/dist/evaluation/hooks/langfuseAdapter.js +172 -0
- package/dist/evaluation/hooks/observabilityHooks.d.ts +129 -0
- package/dist/evaluation/hooks/observabilityHooks.js +181 -0
- package/dist/evaluation/index.d.ts +11 -2
- package/dist/evaluation/index.js +15 -0
- package/dist/evaluation/pipeline/evaluationPipeline.d.ts +114 -0
- package/dist/evaluation/pipeline/evaluationPipeline.js +381 -0
- package/dist/evaluation/pipeline/index.d.ts +8 -0
- package/dist/evaluation/pipeline/index.js +8 -0
- package/dist/evaluation/pipeline/pipelineBuilder.d.ts +126 -0
- package/dist/evaluation/pipeline/pipelineBuilder.js +260 -0
- package/dist/evaluation/pipeline/presets.d.ts +66 -0
- package/dist/evaluation/pipeline/presets.js +224 -0
- package/dist/evaluation/pipeline/strategies/batchStrategy.d.ts +99 -0
- package/dist/evaluation/pipeline/strategies/batchStrategy.js +238 -0
- package/dist/evaluation/pipeline/strategies/index.d.ts +6 -0
- package/dist/evaluation/pipeline/strategies/index.js +6 -0
- package/dist/evaluation/pipeline/strategies/samplingStrategy.d.ts +76 -0
- package/dist/evaluation/pipeline/strategies/samplingStrategy.js +238 -0
- package/dist/evaluation/reporting/index.d.ts +6 -0
- package/dist/evaluation/reporting/index.js +6 -0
- package/dist/evaluation/reporting/metricsCollector.d.ts +147 -0
- package/dist/evaluation/reporting/metricsCollector.js +285 -0
- package/dist/evaluation/reporting/reportGenerator.d.ts +90 -0
- package/dist/evaluation/reporting/reportGenerator.js +374 -0
- package/dist/evaluation/scorers/baseScorer.d.ts +83 -0
- package/dist/evaluation/scorers/baseScorer.js +232 -0
- package/dist/evaluation/scorers/customScorerUtils.d.ts +95 -0
- package/dist/evaluation/scorers/customScorerUtils.js +381 -0
- package/dist/evaluation/scorers/index.d.ts +10 -0
- package/dist/evaluation/scorers/index.js +16 -0
- package/dist/evaluation/scorers/llm/answerRelevancyScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/answerRelevancyScorer.js +99 -0
- package/dist/evaluation/scorers/llm/baseLLMScorer.d.ts +71 -0
- package/dist/evaluation/scorers/llm/baseLLMScorer.js +281 -0
- package/dist/evaluation/scorers/llm/biasDetectionScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/biasDetectionScorer.js +127 -0
- package/dist/evaluation/scorers/llm/contextPrecisionScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/contextPrecisionScorer.js +92 -0
- package/dist/evaluation/scorers/llm/contextRelevancyScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/contextRelevancyScorer.js +107 -0
- package/dist/evaluation/scorers/llm/faithfulnessScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/faithfulnessScorer.js +121 -0
- package/dist/evaluation/scorers/llm/hallucinationScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/hallucinationScorer.js +140 -0
- package/dist/evaluation/scorers/llm/index.d.ts +15 -0
- package/dist/evaluation/scorers/llm/index.js +16 -0
- package/dist/evaluation/scorers/llm/promptAlignmentScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/promptAlignmentScorer.js +106 -0
- package/dist/evaluation/scorers/llm/summarizationScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/summarizationScorer.js +114 -0
- package/dist/evaluation/scorers/llm/toneConsistencyScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/toneConsistencyScorer.js +106 -0
- package/dist/evaluation/scorers/llm/toxicityScorer.d.ts +12 -0
- package/dist/evaluation/scorers/llm/toxicityScorer.js +121 -0
- package/dist/evaluation/scorers/rule/baseRuleScorer.d.ts +77 -0
- package/dist/evaluation/scorers/rule/baseRuleScorer.js +233 -0
- package/dist/evaluation/scorers/rule/contentSimilarityScorer.d.ts +108 -0
- package/dist/evaluation/scorers/rule/contentSimilarityScorer.js +350 -0
- package/dist/evaluation/scorers/rule/formatScorer.d.ts +147 -0
- package/dist/evaluation/scorers/rule/formatScorer.js +470 -0
- package/dist/evaluation/scorers/rule/index.d.ts +9 -0
- package/dist/evaluation/scorers/rule/index.js +10 -0
- package/dist/evaluation/scorers/rule/keywordCoverageScorer.d.ts +83 -0
- package/dist/evaluation/scorers/rule/keywordCoverageScorer.js +347 -0
- package/dist/evaluation/scorers/rule/lengthScorer.d.ts +105 -0
- package/dist/evaluation/scorers/rule/lengthScorer.js +351 -0
- package/dist/evaluation/scorers/scorerBuilder.d.ts +161 -0
- package/dist/evaluation/scorers/scorerBuilder.js +420 -0
- package/dist/evaluation/scorers/scorerRegistry.d.ts +62 -0
- package/dist/evaluation/scorers/scorerRegistry.js +467 -0
- package/dist/index.d.ts +37 -25
- package/dist/index.js +65 -26
- package/dist/lib/auth/providers/BaseAuthProvider.d.ts +1 -1
- package/dist/lib/evaluation/BatchEvaluator.d.ts +163 -0
- package/dist/lib/evaluation/BatchEvaluator.js +268 -0
- package/dist/lib/evaluation/EvaluationAggregator.d.ts +272 -0
- package/dist/lib/evaluation/EvaluationAggregator.js +378 -0
- package/dist/lib/evaluation/EvaluatorFactory.d.ts +113 -0
- package/dist/lib/evaluation/EvaluatorFactory.js +281 -0
- package/dist/lib/evaluation/EvaluatorRegistry.d.ts +160 -0
- package/dist/lib/evaluation/EvaluatorRegistry.js +185 -0
- package/dist/lib/evaluation/errors/EvaluationError.d.ts +189 -0
- package/dist/lib/evaluation/errors/EvaluationError.js +207 -0
- package/dist/lib/evaluation/errors/index.d.ts +4 -0
- package/dist/lib/evaluation/errors/index.js +5 -0
- package/dist/lib/evaluation/hooks/index.d.ts +6 -0
- package/dist/lib/evaluation/hooks/index.js +7 -0
- package/dist/lib/evaluation/hooks/langfuseAdapter.d.ts +99 -0
- package/dist/lib/evaluation/hooks/langfuseAdapter.js +173 -0
- package/dist/lib/evaluation/hooks/observabilityHooks.d.ts +129 -0
- package/dist/lib/evaluation/hooks/observabilityHooks.js +182 -0
- package/dist/lib/evaluation/index.d.ts +11 -2
- package/dist/lib/evaluation/index.js +15 -0
- package/dist/lib/evaluation/pipeline/evaluationPipeline.d.ts +114 -0
- package/dist/lib/evaluation/pipeline/evaluationPipeline.js +382 -0
- package/dist/lib/evaluation/pipeline/index.d.ts +8 -0
- package/dist/lib/evaluation/pipeline/index.js +9 -0
- package/dist/lib/evaluation/pipeline/pipelineBuilder.d.ts +126 -0
- package/dist/lib/evaluation/pipeline/pipelineBuilder.js +261 -0
- package/dist/lib/evaluation/pipeline/presets.d.ts +66 -0
- package/dist/lib/evaluation/pipeline/presets.js +225 -0
- package/dist/lib/evaluation/pipeline/strategies/batchStrategy.d.ts +99 -0
- package/dist/lib/evaluation/pipeline/strategies/batchStrategy.js +239 -0
- package/dist/lib/evaluation/pipeline/strategies/index.d.ts +6 -0
- package/dist/lib/evaluation/pipeline/strategies/index.js +7 -0
- package/dist/lib/evaluation/pipeline/strategies/samplingStrategy.d.ts +76 -0
- package/dist/lib/evaluation/pipeline/strategies/samplingStrategy.js +239 -0
- package/dist/lib/evaluation/reporting/index.d.ts +6 -0
- package/dist/lib/evaluation/reporting/index.js +7 -0
- package/dist/lib/evaluation/reporting/metricsCollector.d.ts +147 -0
- package/dist/lib/evaluation/reporting/metricsCollector.js +286 -0
- package/dist/lib/evaluation/reporting/reportGenerator.d.ts +90 -0
- package/dist/lib/evaluation/reporting/reportGenerator.js +375 -0
- package/dist/lib/evaluation/scorers/baseScorer.d.ts +83 -0
- package/dist/lib/evaluation/scorers/baseScorer.js +233 -0
- package/dist/lib/evaluation/scorers/customScorerUtils.d.ts +95 -0
- package/dist/lib/evaluation/scorers/customScorerUtils.js +382 -0
- package/dist/lib/evaluation/scorers/index.d.ts +10 -0
- package/dist/lib/evaluation/scorers/index.js +17 -0
- package/dist/lib/evaluation/scorers/llm/answerRelevancyScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/answerRelevancyScorer.js +100 -0
- package/dist/lib/evaluation/scorers/llm/baseLLMScorer.d.ts +71 -0
- package/dist/lib/evaluation/scorers/llm/baseLLMScorer.js +282 -0
- package/dist/lib/evaluation/scorers/llm/biasDetectionScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/biasDetectionScorer.js +128 -0
- package/dist/lib/evaluation/scorers/llm/contextPrecisionScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/contextPrecisionScorer.js +93 -0
- package/dist/lib/evaluation/scorers/llm/contextRelevancyScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/contextRelevancyScorer.js +108 -0
- package/dist/lib/evaluation/scorers/llm/faithfulnessScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/faithfulnessScorer.js +122 -0
- package/dist/lib/evaluation/scorers/llm/hallucinationScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/hallucinationScorer.js +141 -0
- package/dist/lib/evaluation/scorers/llm/index.d.ts +15 -0
- package/dist/lib/evaluation/scorers/llm/index.js +17 -0
- package/dist/lib/evaluation/scorers/llm/promptAlignmentScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/promptAlignmentScorer.js +107 -0
- package/dist/lib/evaluation/scorers/llm/summarizationScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/summarizationScorer.js +115 -0
- package/dist/lib/evaluation/scorers/llm/toneConsistencyScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/toneConsistencyScorer.js +107 -0
- package/dist/lib/evaluation/scorers/llm/toxicityScorer.d.ts +12 -0
- package/dist/lib/evaluation/scorers/llm/toxicityScorer.js +122 -0
- package/dist/lib/evaluation/scorers/rule/baseRuleScorer.d.ts +77 -0
- package/dist/lib/evaluation/scorers/rule/baseRuleScorer.js +234 -0
- package/dist/lib/evaluation/scorers/rule/contentSimilarityScorer.d.ts +108 -0
- package/dist/lib/evaluation/scorers/rule/contentSimilarityScorer.js +351 -0
- package/dist/lib/evaluation/scorers/rule/formatScorer.d.ts +147 -0
- package/dist/lib/evaluation/scorers/rule/formatScorer.js +471 -0
- package/dist/lib/evaluation/scorers/rule/index.d.ts +9 -0
- package/dist/lib/evaluation/scorers/rule/index.js +11 -0
- package/dist/lib/evaluation/scorers/rule/keywordCoverageScorer.d.ts +83 -0
- package/dist/lib/evaluation/scorers/rule/keywordCoverageScorer.js +348 -0
- package/dist/lib/evaluation/scorers/rule/lengthScorer.d.ts +105 -0
- package/dist/lib/evaluation/scorers/rule/lengthScorer.js +352 -0
- package/dist/lib/evaluation/scorers/scorerBuilder.d.ts +161 -0
- package/dist/lib/evaluation/scorers/scorerBuilder.js +421 -0
- package/dist/lib/evaluation/scorers/scorerRegistry.d.ts +62 -0
- package/dist/lib/evaluation/scorers/scorerRegistry.js +468 -0
- package/dist/lib/index.d.ts +37 -25
- package/dist/lib/index.js +65 -26
- package/dist/lib/neurolink.d.ts +204 -0
- package/dist/lib/neurolink.js +296 -0
- package/dist/lib/processors/media/VideoProcessor.d.ts +8 -2
- package/dist/lib/processors/media/VideoProcessor.js +90 -41
- package/dist/lib/telemetry/telemetryService.d.ts +1 -1
- package/dist/lib/telemetry/telemetryService.js +27 -13
- package/dist/lib/types/index.d.ts +3 -1
- package/dist/lib/types/index.js +3 -2
- package/dist/lib/types/scorerTypes.d.ts +423 -0
- package/dist/lib/types/scorerTypes.js +6 -0
- package/dist/lib/utils/errorHandling.d.ts +20 -0
- package/dist/lib/utils/errorHandling.js +60 -0
- package/dist/neurolink.d.ts +204 -0
- package/dist/neurolink.js +296 -0
- package/dist/processors/media/VideoProcessor.d.ts +8 -2
- package/dist/processors/media/VideoProcessor.js +90 -41
- package/dist/telemetry/telemetryService.d.ts +1 -1
- package/dist/telemetry/telemetryService.js +27 -13
- package/dist/types/index.d.ts +3 -1
- package/dist/types/index.js +3 -2
- package/dist/types/scorerTypes.d.ts +423 -0
- package/dist/types/scorerTypes.js +5 -0
- package/dist/utils/errorHandling.d.ts +20 -0
- package/dist/utils/errorHandling.js +60 -0
- package/package.json +7 -7
- package/dist/processors/media/ffprobe-static.d.ts +0 -4
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file EvaluatorRegistry - Registry for evaluation strategies.
|
|
3
|
+
* Extends BaseRegistry to provide dynamic strategy registration and lookup.
|
|
4
|
+
*/
|
|
5
|
+
import { BaseRegistry } from "../core/infrastructure/index.js";
|
|
6
|
+
import { createStrategyNotFoundError } from "./errors/EvaluationError.js";
|
|
7
|
+
import { withTimeout, ErrorFactory } from "../utils/errorHandling.js";
|
|
8
|
+
/**
|
|
9
|
+
* Registry for evaluation strategies.
|
|
10
|
+
* Allows dynamic registration and retrieval of evaluation strategies.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* // Register a custom strategy
|
|
15
|
+
* EvaluatorRegistry.getInstance().registerStrategy(
|
|
16
|
+
* 'custom-ragas',
|
|
17
|
+
* async () => ({
|
|
18
|
+
* evaluate: async (options, result) => { ... }
|
|
19
|
+
* }),
|
|
20
|
+
* {
|
|
21
|
+
* name: 'Custom RAGAS',
|
|
22
|
+
* description: 'Custom RAGAS implementation',
|
|
23
|
+
* requiresLLM: true,
|
|
24
|
+
* defaultModel: 'gpt-4',
|
|
25
|
+
* defaultProvider: 'openai',
|
|
26
|
+
* version: '1.0.0',
|
|
27
|
+
* features: ['custom-metrics']
|
|
28
|
+
* }
|
|
29
|
+
* );
|
|
30
|
+
*
|
|
31
|
+
* // Get a strategy
|
|
32
|
+
* const strategy = await EvaluatorRegistry.getInstance().getStrategy('ragas');
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export class EvaluatorRegistry extends BaseRegistry {
|
|
36
|
+
static instance = null;
|
|
37
|
+
constructor() {
|
|
38
|
+
super();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Gets the singleton instance of the EvaluatorRegistry.
|
|
42
|
+
*/
|
|
43
|
+
static getInstance() {
|
|
44
|
+
if (!EvaluatorRegistry.instance) {
|
|
45
|
+
EvaluatorRegistry.instance = new EvaluatorRegistry();
|
|
46
|
+
}
|
|
47
|
+
return EvaluatorRegistry.instance;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Resets the singleton instance (useful for testing).
|
|
51
|
+
*/
|
|
52
|
+
static resetInstance() {
|
|
53
|
+
EvaluatorRegistry.instance = null;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Registers all built-in evaluation strategies.
|
|
57
|
+
* This is called automatically on first access.
|
|
58
|
+
*/
|
|
59
|
+
async registerAll() {
|
|
60
|
+
// Register the built-in RAGAS strategy
|
|
61
|
+
this.registerStrategy("ragas", async () => {
|
|
62
|
+
const { RAGASEvaluator } = await import("./ragasEvaluator.js");
|
|
63
|
+
const { ContextBuilder } = await import("./contextBuilder.js");
|
|
64
|
+
return async (options, result, config) => {
|
|
65
|
+
const contextBuilder = new ContextBuilder();
|
|
66
|
+
const ragasEvaluator = new RAGASEvaluator(config?.evaluationModel, config?.provider, config?.threshold, config?.promptGenerator);
|
|
67
|
+
const evalContext = contextBuilder.buildContext(options, result);
|
|
68
|
+
const evaluationTimeoutMs = config?.options?.timeout ?? 60000;
|
|
69
|
+
const evaluationResult = await withTimeout(ragasEvaluator.evaluate(evalContext), evaluationTimeoutMs, ErrorFactory.evaluationTimeout("strategy evaluation", evaluationTimeoutMs));
|
|
70
|
+
return { evaluationResult, evalContext };
|
|
71
|
+
};
|
|
72
|
+
}, {
|
|
73
|
+
name: "RAGAS Evaluator",
|
|
74
|
+
description: "RAGAS-style LLM-as-judge evaluation with relevance, accuracy, and completeness metrics",
|
|
75
|
+
requiresLLM: true,
|
|
76
|
+
defaultModel: "gemini-1.5-flash",
|
|
77
|
+
defaultProvider: "vertex",
|
|
78
|
+
version: "1.0.0",
|
|
79
|
+
features: [
|
|
80
|
+
"relevance-scoring",
|
|
81
|
+
"accuracy-scoring",
|
|
82
|
+
"completeness-scoring",
|
|
83
|
+
"reasoning",
|
|
84
|
+
"suggestions",
|
|
85
|
+
],
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Registers an evaluation strategy with the registry.
|
|
90
|
+
*
|
|
91
|
+
* @param id - Unique identifier for the strategy
|
|
92
|
+
* @param factory - Factory function that creates the strategy
|
|
93
|
+
* @param metadata - Metadata about the strategy
|
|
94
|
+
*/
|
|
95
|
+
registerStrategy(id, factory, metadata) {
|
|
96
|
+
this.register(id, factory, [], { metadata });
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Gets an evaluation strategy by ID.
|
|
100
|
+
*
|
|
101
|
+
* @param id - The strategy identifier
|
|
102
|
+
* @returns The evaluation strategy function
|
|
103
|
+
* @throws {NeuroLinkFeatureError} If the strategy is not found
|
|
104
|
+
*/
|
|
105
|
+
async getStrategy(id) {
|
|
106
|
+
const strategy = await this.get(id);
|
|
107
|
+
if (!strategy) {
|
|
108
|
+
const strategies = await this.listStrategies();
|
|
109
|
+
const available = strategies.map((s) => s.id);
|
|
110
|
+
throw createStrategyNotFoundError(id, available);
|
|
111
|
+
}
|
|
112
|
+
return strategy;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Checks if a strategy exists in the registry.
|
|
116
|
+
*
|
|
117
|
+
* @param id - The strategy identifier
|
|
118
|
+
* @returns true if the strategy exists
|
|
119
|
+
*/
|
|
120
|
+
async hasStrategy(id) {
|
|
121
|
+
await this.ensureInitialized();
|
|
122
|
+
return this.has(id);
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Lists all registered strategies with their metadata.
|
|
126
|
+
*
|
|
127
|
+
* @returns Array of strategy IDs and their metadata
|
|
128
|
+
*/
|
|
129
|
+
async listStrategies() {
|
|
130
|
+
await this.ensureInitialized();
|
|
131
|
+
return this.list();
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Gets the metadata for a specific strategy.
|
|
135
|
+
*
|
|
136
|
+
* @param id - The strategy identifier
|
|
137
|
+
* @returns The strategy metadata or undefined if not found
|
|
138
|
+
*/
|
|
139
|
+
async getStrategyMetadata(id) {
|
|
140
|
+
await this.ensureInitialized();
|
|
141
|
+
const entry = this.items.get(id);
|
|
142
|
+
return entry?.metadata;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Unregisters a strategy from the registry.
|
|
146
|
+
*
|
|
147
|
+
* @param id - The strategy identifier
|
|
148
|
+
* @returns true if the strategy was removed, false if it didn't exist
|
|
149
|
+
*/
|
|
150
|
+
async unregisterStrategy(id) {
|
|
151
|
+
await this.ensureInitialized();
|
|
152
|
+
if (this.items.has(id)) {
|
|
153
|
+
this.items.delete(id);
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
return false;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Gets strategies that support a specific feature.
|
|
160
|
+
*
|
|
161
|
+
* @param feature - The feature to filter by
|
|
162
|
+
* @returns Array of strategy IDs that support the feature
|
|
163
|
+
*/
|
|
164
|
+
async getStrategiesWithFeature(feature) {
|
|
165
|
+
await this.ensureInitialized();
|
|
166
|
+
return this.list()
|
|
167
|
+
.filter((item) => item.metadata.features.includes(feature))
|
|
168
|
+
.map((item) => item.id);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Gets strategies that use a specific provider.
|
|
172
|
+
*
|
|
173
|
+
* @param provider - The provider to filter by
|
|
174
|
+
* @returns Array of strategy IDs that use the provider
|
|
175
|
+
*/
|
|
176
|
+
async getStrategiesByProvider(provider) {
|
|
177
|
+
await this.ensureInitialized();
|
|
178
|
+
return this.list()
|
|
179
|
+
.filter((item) => item.metadata.defaultProvider === provider)
|
|
180
|
+
.map((item) => item.id);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
// Export singleton instance getter for convenience
|
|
184
|
+
export const getEvaluatorRegistry = () => EvaluatorRegistry.getInstance();
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Typed error hierarchy for the Evaluation/Scoring System.
|
|
3
|
+
* Uses NeuroLinkFeatureError and createErrorFactory from core infrastructure.
|
|
4
|
+
*/
|
|
5
|
+
import { NeuroLinkFeatureError } from "../../core/infrastructure/index.js";
|
|
6
|
+
import type { EnhancedEvaluationContext } from "../../types/evaluationTypes.js";
|
|
7
|
+
/**
|
|
8
|
+
* Error codes for the Evaluation feature.
|
|
9
|
+
* These codes help identify specific error scenarios for proper handling.
|
|
10
|
+
*/
|
|
11
|
+
export declare const EvaluationErrorCodes: {
|
|
12
|
+
/** The evaluation process itself failed */
|
|
13
|
+
readonly EVALUATION_FAILED: "EVALUATION_FAILED";
|
|
14
|
+
/** Failed to parse the evaluation response from the judge LLM */
|
|
15
|
+
readonly PARSE_ERROR: "PARSE_ERROR";
|
|
16
|
+
/** The requested evaluation strategy was not found in the registry */
|
|
17
|
+
readonly STRATEGY_NOT_FOUND: "STRATEGY_NOT_FOUND";
|
|
18
|
+
/** The AI provider for evaluation failed */
|
|
19
|
+
readonly PROVIDER_ERROR: "PROVIDER_ERROR";
|
|
20
|
+
/** Configuration for evaluation is invalid or missing */
|
|
21
|
+
readonly CONFIGURATION_ERROR: "CONFIGURATION_ERROR";
|
|
22
|
+
/** The custom evaluator function threw an error */
|
|
23
|
+
readonly CUSTOM_EVALUATOR_ERROR: "CUSTOM_EVALUATOR_ERROR";
|
|
24
|
+
/** Batch evaluation failed for one or more items */
|
|
25
|
+
readonly BATCH_EVALUATION_ERROR: "BATCH_EVALUATION_ERROR";
|
|
26
|
+
/** Aggregation of evaluation results failed */
|
|
27
|
+
readonly AGGREGATION_ERROR: "AGGREGATION_ERROR";
|
|
28
|
+
/** Registry operation failed */
|
|
29
|
+
readonly REGISTRY_ERROR: "REGISTRY_ERROR";
|
|
30
|
+
/** Maximum retry attempts exceeded */
|
|
31
|
+
readonly MAX_RETRIES_EXCEEDED: "MAX_RETRIES_EXCEEDED";
|
|
32
|
+
/** Timeout during evaluation */
|
|
33
|
+
readonly TIMEOUT_ERROR: "TIMEOUT_ERROR";
|
|
34
|
+
/** Rate limit hit during evaluation */
|
|
35
|
+
readonly RATE_LIMIT_ERROR: "RATE_LIMIT_ERROR";
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Type for evaluation error codes
|
|
39
|
+
*/
|
|
40
|
+
export type EvaluationErrorCode = (typeof EvaluationErrorCodes)[keyof typeof EvaluationErrorCodes];
|
|
41
|
+
/**
|
|
42
|
+
* Factory for creating typed evaluation errors.
|
|
43
|
+
* Uses the createErrorFactory pattern from core infrastructure.
|
|
44
|
+
*/
|
|
45
|
+
export declare const evaluationErrors: {
|
|
46
|
+
codes: {
|
|
47
|
+
/** The evaluation process itself failed */
|
|
48
|
+
readonly EVALUATION_FAILED: "EVALUATION_FAILED";
|
|
49
|
+
/** Failed to parse the evaluation response from the judge LLM */
|
|
50
|
+
readonly PARSE_ERROR: "PARSE_ERROR";
|
|
51
|
+
/** The requested evaluation strategy was not found in the registry */
|
|
52
|
+
readonly STRATEGY_NOT_FOUND: "STRATEGY_NOT_FOUND";
|
|
53
|
+
/** The AI provider for evaluation failed */
|
|
54
|
+
readonly PROVIDER_ERROR: "PROVIDER_ERROR";
|
|
55
|
+
/** Configuration for evaluation is invalid or missing */
|
|
56
|
+
readonly CONFIGURATION_ERROR: "CONFIGURATION_ERROR";
|
|
57
|
+
/** The custom evaluator function threw an error */
|
|
58
|
+
readonly CUSTOM_EVALUATOR_ERROR: "CUSTOM_EVALUATOR_ERROR";
|
|
59
|
+
/** Batch evaluation failed for one or more items */
|
|
60
|
+
readonly BATCH_EVALUATION_ERROR: "BATCH_EVALUATION_ERROR";
|
|
61
|
+
/** Aggregation of evaluation results failed */
|
|
62
|
+
readonly AGGREGATION_ERROR: "AGGREGATION_ERROR";
|
|
63
|
+
/** Registry operation failed */
|
|
64
|
+
readonly REGISTRY_ERROR: "REGISTRY_ERROR";
|
|
65
|
+
/** Maximum retry attempts exceeded */
|
|
66
|
+
readonly MAX_RETRIES_EXCEEDED: "MAX_RETRIES_EXCEEDED";
|
|
67
|
+
/** Timeout during evaluation */
|
|
68
|
+
readonly TIMEOUT_ERROR: "TIMEOUT_ERROR";
|
|
69
|
+
/** Rate limit hit during evaluation */
|
|
70
|
+
readonly RATE_LIMIT_ERROR: "RATE_LIMIT_ERROR";
|
|
71
|
+
};
|
|
72
|
+
create: (code: "PROVIDER_ERROR" | "CONFIGURATION_ERROR" | "EVALUATION_FAILED" | "PARSE_ERROR" | "STRATEGY_NOT_FOUND" | "CUSTOM_EVALUATOR_ERROR" | "BATCH_EVALUATION_ERROR" | "AGGREGATION_ERROR" | "REGISTRY_ERROR" | "MAX_RETRIES_EXCEEDED" | "TIMEOUT_ERROR" | "RATE_LIMIT_ERROR", message: string, options?: {
|
|
73
|
+
retryable?: boolean;
|
|
74
|
+
details?: Record<string, unknown>;
|
|
75
|
+
cause?: Error;
|
|
76
|
+
} | undefined) => NeuroLinkFeatureError;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Extended evaluation context for error details.
|
|
80
|
+
* Provides rich debugging information when errors occur.
|
|
81
|
+
*/
|
|
82
|
+
export interface EvaluationErrorContext {
|
|
83
|
+
/** Length of the user query (redacted for safety) */
|
|
84
|
+
userQueryLength?: number;
|
|
85
|
+
/** Length of the AI response (redacted for safety) */
|
|
86
|
+
aiResponseLength?: number;
|
|
87
|
+
/** The current attempt number */
|
|
88
|
+
attemptNumber?: number;
|
|
89
|
+
/** Previous evaluation scores if any */
|
|
90
|
+
previousScores?: number[];
|
|
91
|
+
/** The evaluation strategy in use */
|
|
92
|
+
strategy?: string;
|
|
93
|
+
/** The evaluation model being used */
|
|
94
|
+
evaluationModel?: string;
|
|
95
|
+
/** The provider being used */
|
|
96
|
+
provider?: string;
|
|
97
|
+
/** Length of the raw response (redacted for safety) */
|
|
98
|
+
rawResponseLength?: number;
|
|
99
|
+
/** Any additional context */
|
|
100
|
+
additionalContext?: Record<string, unknown>;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Checks if an error is retryable based on its code.
|
|
104
|
+
* Transient errors (timeout, rate limit, some provider errors) are retryable.
|
|
105
|
+
*
|
|
106
|
+
* @param error - The error to check
|
|
107
|
+
* @returns true if the error is retryable
|
|
108
|
+
*/
|
|
109
|
+
export declare function isRetryableEvaluationError(error: NeuroLinkFeatureError): boolean;
|
|
110
|
+
/**
|
|
111
|
+
* Checks if an error is a NeuroLinkFeatureError from the Evaluation feature.
|
|
112
|
+
*
|
|
113
|
+
* @param error - The error to check
|
|
114
|
+
* @returns true if the error is an evaluation error
|
|
115
|
+
*/
|
|
116
|
+
export declare function isEvaluationError(error: unknown): error is NeuroLinkFeatureError;
|
|
117
|
+
/**
|
|
118
|
+
* Helper function to create an evaluation failed error with context.
|
|
119
|
+
*
|
|
120
|
+
* @param message - The error message
|
|
121
|
+
* @param context - The evaluation context
|
|
122
|
+
* @param cause - The underlying cause error
|
|
123
|
+
* @returns A typed NeuroLinkFeatureError
|
|
124
|
+
*/
|
|
125
|
+
export declare function createEvaluationFailedError(message: string, context?: EvaluationErrorContext, cause?: Error): NeuroLinkFeatureError;
|
|
126
|
+
/**
|
|
127
|
+
* Helper function to create a parse error with raw response.
|
|
128
|
+
*
|
|
129
|
+
* @param rawResponse - The raw response that failed to parse
|
|
130
|
+
* @param cause - The underlying parse error
|
|
131
|
+
* @returns A typed NeuroLinkFeatureError
|
|
132
|
+
*/
|
|
133
|
+
export declare function createParseError(rawResponse: string, cause?: Error): NeuroLinkFeatureError;
|
|
134
|
+
/**
|
|
135
|
+
* Helper function to create a strategy not found error.
|
|
136
|
+
*
|
|
137
|
+
* @param strategyName - The name of the strategy that was not found
|
|
138
|
+
* @param availableStrategies - List of available strategies
|
|
139
|
+
* @returns A typed NeuroLinkFeatureError
|
|
140
|
+
*/
|
|
141
|
+
export declare function createStrategyNotFoundError(strategyName: string, availableStrategies?: string[]): NeuroLinkFeatureError;
|
|
142
|
+
/**
|
|
143
|
+
* Helper function to create a provider error.
|
|
144
|
+
*
|
|
145
|
+
* @param message - The error message
|
|
146
|
+
* @param provider - The provider that failed
|
|
147
|
+
* @param cause - The underlying cause error
|
|
148
|
+
* @returns A typed NeuroLinkFeatureError
|
|
149
|
+
*/
|
|
150
|
+
export declare function createProviderError(message: string, provider: string, cause?: Error, options?: {
|
|
151
|
+
retryable?: boolean;
|
|
152
|
+
}): NeuroLinkFeatureError;
|
|
153
|
+
/**
|
|
154
|
+
* Helper function to create a max retries exceeded error.
|
|
155
|
+
*
|
|
156
|
+
* @param attempts - The number of attempts made
|
|
157
|
+
* @param lastScore - The last evaluation score
|
|
158
|
+
* @param threshold - The passing threshold
|
|
159
|
+
* @param context - The evaluation context
|
|
160
|
+
* @returns A typed NeuroLinkFeatureError
|
|
161
|
+
*/
|
|
162
|
+
export declare function createMaxRetriesExceededError(attempts: number, lastScore: number, threshold: number, context?: EvaluationErrorContext): NeuroLinkFeatureError;
|
|
163
|
+
/**
|
|
164
|
+
* Helper function to create a batch evaluation error.
|
|
165
|
+
*
|
|
166
|
+
* @param failedCount - Number of evaluations that failed
|
|
167
|
+
* @param totalCount - Total number of evaluations attempted
|
|
168
|
+
* @param errors - Array of individual errors
|
|
169
|
+
* @returns A typed NeuroLinkFeatureError
|
|
170
|
+
*/
|
|
171
|
+
export declare function createBatchEvaluationError(failedCount: number, totalCount: number, errors: Array<{
|
|
172
|
+
index: number;
|
|
173
|
+
error: Error;
|
|
174
|
+
}>): NeuroLinkFeatureError;
|
|
175
|
+
/**
|
|
176
|
+
* Helper function to create a configuration error.
|
|
177
|
+
*
|
|
178
|
+
* @param message - The error message
|
|
179
|
+
* @param configIssue - Description of the configuration issue
|
|
180
|
+
* @returns A typed NeuroLinkFeatureError
|
|
181
|
+
*/
|
|
182
|
+
export declare function createConfigurationError(message: string, configIssue: string): NeuroLinkFeatureError;
|
|
183
|
+
/**
|
|
184
|
+
* Converts an evaluation context to error context for debugging.
|
|
185
|
+
*
|
|
186
|
+
* @param context - The enhanced evaluation context
|
|
187
|
+
* @returns An EvaluationErrorContext
|
|
188
|
+
*/
|
|
189
|
+
export declare function contextToErrorContext(context: EnhancedEvaluationContext): EvaluationErrorContext;
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Typed error hierarchy for the Evaluation/Scoring System.
|
|
3
|
+
* Uses NeuroLinkFeatureError and createErrorFactory from core infrastructure.
|
|
4
|
+
*/
|
|
5
|
+
import { NeuroLinkFeatureError, createErrorFactory, } from "../../core/infrastructure/index.js";
|
|
6
|
+
/**
|
|
7
|
+
* Error codes for the Evaluation feature.
|
|
8
|
+
* These codes help identify specific error scenarios for proper handling.
|
|
9
|
+
*/
|
|
10
|
+
export const EvaluationErrorCodes = {
|
|
11
|
+
/** The evaluation process itself failed */
|
|
12
|
+
EVALUATION_FAILED: "EVALUATION_FAILED",
|
|
13
|
+
/** Failed to parse the evaluation response from the judge LLM */
|
|
14
|
+
PARSE_ERROR: "PARSE_ERROR",
|
|
15
|
+
/** The requested evaluation strategy was not found in the registry */
|
|
16
|
+
STRATEGY_NOT_FOUND: "STRATEGY_NOT_FOUND",
|
|
17
|
+
/** The AI provider for evaluation failed */
|
|
18
|
+
PROVIDER_ERROR: "PROVIDER_ERROR",
|
|
19
|
+
/** Configuration for evaluation is invalid or missing */
|
|
20
|
+
CONFIGURATION_ERROR: "CONFIGURATION_ERROR",
|
|
21
|
+
/** The custom evaluator function threw an error */
|
|
22
|
+
CUSTOM_EVALUATOR_ERROR: "CUSTOM_EVALUATOR_ERROR",
|
|
23
|
+
/** Batch evaluation failed for one or more items */
|
|
24
|
+
BATCH_EVALUATION_ERROR: "BATCH_EVALUATION_ERROR",
|
|
25
|
+
/** Aggregation of evaluation results failed */
|
|
26
|
+
AGGREGATION_ERROR: "AGGREGATION_ERROR",
|
|
27
|
+
/** Registry operation failed */
|
|
28
|
+
REGISTRY_ERROR: "REGISTRY_ERROR",
|
|
29
|
+
/** Maximum retry attempts exceeded */
|
|
30
|
+
MAX_RETRIES_EXCEEDED: "MAX_RETRIES_EXCEEDED",
|
|
31
|
+
/** Timeout during evaluation */
|
|
32
|
+
TIMEOUT_ERROR: "TIMEOUT_ERROR",
|
|
33
|
+
/** Rate limit hit during evaluation */
|
|
34
|
+
RATE_LIMIT_ERROR: "RATE_LIMIT_ERROR",
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Factory for creating typed evaluation errors.
|
|
38
|
+
* Uses the createErrorFactory pattern from core infrastructure.
|
|
39
|
+
*/
|
|
40
|
+
export const evaluationErrors = createErrorFactory("Evaluation", EvaluationErrorCodes);
|
|
41
|
+
/**
|
|
42
|
+
* Checks if an error is retryable based on its code.
|
|
43
|
+
* Transient errors (timeout, rate limit, some provider errors) are retryable.
|
|
44
|
+
*
|
|
45
|
+
* @param error - The error to check
|
|
46
|
+
* @returns true if the error is retryable
|
|
47
|
+
*/
|
|
48
|
+
export function isRetryableEvaluationError(error) {
|
|
49
|
+
if (error.retryable) {
|
|
50
|
+
return true;
|
|
51
|
+
}
|
|
52
|
+
const retryableCodes = [
|
|
53
|
+
EvaluationErrorCodes.TIMEOUT_ERROR,
|
|
54
|
+
EvaluationErrorCodes.RATE_LIMIT_ERROR,
|
|
55
|
+
];
|
|
56
|
+
return retryableCodes.includes(error.code);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Checks if an error is a NeuroLinkFeatureError from the Evaluation feature.
|
|
60
|
+
*
|
|
61
|
+
* @param error - The error to check
|
|
62
|
+
* @returns true if the error is an evaluation error
|
|
63
|
+
*/
|
|
64
|
+
export function isEvaluationError(error) {
|
|
65
|
+
return (error instanceof NeuroLinkFeatureError && error.feature === "Evaluation");
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Helper function to create an evaluation failed error with context.
|
|
69
|
+
*
|
|
70
|
+
* @param message - The error message
|
|
71
|
+
* @param context - The evaluation context
|
|
72
|
+
* @param cause - The underlying cause error
|
|
73
|
+
* @returns A typed NeuroLinkFeatureError
|
|
74
|
+
*/
|
|
75
|
+
export function createEvaluationFailedError(message, context, cause) {
|
|
76
|
+
return evaluationErrors.create("EVALUATION_FAILED", message, {
|
|
77
|
+
retryable: false,
|
|
78
|
+
details: {
|
|
79
|
+
evaluationContext: context,
|
|
80
|
+
},
|
|
81
|
+
cause,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Helper function to create a parse error with raw response.
|
|
86
|
+
*
|
|
87
|
+
* @param rawResponse - The raw response that failed to parse
|
|
88
|
+
* @param cause - The underlying parse error
|
|
89
|
+
* @returns A typed NeuroLinkFeatureError
|
|
90
|
+
*/
|
|
91
|
+
export function createParseError(rawResponse, cause) {
|
|
92
|
+
return evaluationErrors.create("PARSE_ERROR", "Failed to parse evaluation response from judge LLM", {
|
|
93
|
+
retryable: false,
|
|
94
|
+
details: {
|
|
95
|
+
rawResponseLength: rawResponse.length,
|
|
96
|
+
},
|
|
97
|
+
cause,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Helper function to create a strategy not found error.
|
|
102
|
+
*
|
|
103
|
+
* @param strategyName - The name of the strategy that was not found
|
|
104
|
+
* @param availableStrategies - List of available strategies
|
|
105
|
+
* @returns A typed NeuroLinkFeatureError
|
|
106
|
+
*/
|
|
107
|
+
export function createStrategyNotFoundError(strategyName, availableStrategies = []) {
|
|
108
|
+
return evaluationErrors.create("STRATEGY_NOT_FOUND", `Evaluation strategy "${strategyName}" not found in registry`, {
|
|
109
|
+
retryable: false,
|
|
110
|
+
details: {
|
|
111
|
+
requestedStrategy: strategyName,
|
|
112
|
+
availableStrategies,
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Helper function to create a provider error.
|
|
118
|
+
*
|
|
119
|
+
* @param message - The error message
|
|
120
|
+
* @param provider - The provider that failed
|
|
121
|
+
* @param cause - The underlying cause error
|
|
122
|
+
* @returns A typed NeuroLinkFeatureError
|
|
123
|
+
*/
|
|
124
|
+
export function createProviderError(message, provider, cause, options) {
|
|
125
|
+
return evaluationErrors.create("PROVIDER_ERROR", message, {
|
|
126
|
+
retryable: options?.retryable ?? false,
|
|
127
|
+
details: {
|
|
128
|
+
provider,
|
|
129
|
+
errorMessage: cause?.message,
|
|
130
|
+
},
|
|
131
|
+
cause,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Helper function to create a max retries exceeded error.
|
|
136
|
+
*
|
|
137
|
+
* @param attempts - The number of attempts made
|
|
138
|
+
* @param lastScore - The last evaluation score
|
|
139
|
+
* @param threshold - The passing threshold
|
|
140
|
+
* @param context - The evaluation context
|
|
141
|
+
* @returns A typed NeuroLinkFeatureError
|
|
142
|
+
*/
|
|
143
|
+
export function createMaxRetriesExceededError(attempts, lastScore, threshold, context) {
|
|
144
|
+
return evaluationErrors.create("MAX_RETRIES_EXCEEDED", `Maximum retry attempts (${attempts}) exceeded. Last score: ${lastScore}, threshold: ${threshold}`, {
|
|
145
|
+
retryable: false,
|
|
146
|
+
details: {
|
|
147
|
+
attempts,
|
|
148
|
+
lastScore,
|
|
149
|
+
threshold,
|
|
150
|
+
evaluationContext: context,
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Helper function to create a batch evaluation error.
|
|
156
|
+
*
|
|
157
|
+
* @param failedCount - Number of evaluations that failed
|
|
158
|
+
* @param totalCount - Total number of evaluations attempted
|
|
159
|
+
* @param errors - Array of individual errors
|
|
160
|
+
* @returns A typed NeuroLinkFeatureError
|
|
161
|
+
*/
|
|
162
|
+
export function createBatchEvaluationError(failedCount, totalCount, errors) {
|
|
163
|
+
return evaluationErrors.create("BATCH_EVALUATION_ERROR", `Batch evaluation failed: ${failedCount} of ${totalCount} evaluations failed`, {
|
|
164
|
+
retryable: false,
|
|
165
|
+
details: {
|
|
166
|
+
failedCount,
|
|
167
|
+
totalCount,
|
|
168
|
+
successCount: totalCount - failedCount,
|
|
169
|
+
errors: errors.map((e) => ({
|
|
170
|
+
index: e.index,
|
|
171
|
+
message: e.error.message,
|
|
172
|
+
})),
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Helper function to create a configuration error.
|
|
178
|
+
*
|
|
179
|
+
* @param message - The error message
|
|
180
|
+
* @param configIssue - Description of the configuration issue
|
|
181
|
+
* @returns A typed NeuroLinkFeatureError
|
|
182
|
+
*/
|
|
183
|
+
export function createConfigurationError(message, configIssue) {
|
|
184
|
+
return evaluationErrors.create("CONFIGURATION_ERROR", message, {
|
|
185
|
+
retryable: false,
|
|
186
|
+
details: {
|
|
187
|
+
configIssue,
|
|
188
|
+
},
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Converts an evaluation context to error context for debugging.
|
|
193
|
+
*
|
|
194
|
+
* @param context - The enhanced evaluation context
|
|
195
|
+
* @returns An EvaluationErrorContext
|
|
196
|
+
*/
|
|
197
|
+
export function contextToErrorContext(context) {
|
|
198
|
+
return {
|
|
199
|
+
userQueryLength: context.userQuery?.length,
|
|
200
|
+
aiResponseLength: context.aiResponse?.length,
|
|
201
|
+
attemptNumber: context.attemptNumber,
|
|
202
|
+
previousScores: context.previousEvaluations?.map((e) => e.finalScore),
|
|
203
|
+
evaluationModel: undefined, // Will be set by the evaluator
|
|
204
|
+
provider: context.provider,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Exports all evaluation error types and utilities.
|
|
3
|
+
*/
|
|
4
|
+
export { EvaluationErrorCodes, type EvaluationErrorCode, evaluationErrors, type EvaluationErrorContext, isRetryableEvaluationError, isEvaluationError, createEvaluationFailedError, createParseError, createStrategyNotFoundError, createProviderError, createMaxRetriesExceededError, createBatchEvaluationError, createConfigurationError, contextToErrorContext, } from "./EvaluationError.js";
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Exports all evaluation error types and utilities.
|
|
3
|
+
*/
|
|
4
|
+
export { EvaluationErrorCodes, evaluationErrors, isRetryableEvaluationError, isEvaluationError, createEvaluationFailedError, createParseError, createStrategyNotFoundError, createProviderError, createMaxRetriesExceededError, createBatchEvaluationError, createConfigurationError, contextToErrorContext, } from "./EvaluationError.js";
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Hooks Index
|
|
3
|
+
* Export all observability hooks
|
|
4
|
+
*/
|
|
5
|
+
export { createLangfuseAdapter, createMockLangfuseClient, LangfuseAdapter, type LangfuseAdapterConfig, type LangfuseClient, startLangfuseAdapter, } from "./langfuseAdapter.js";
|
|
6
|
+
export { createConsoleLoggerHook, createMetricsCollectorHook, type EvaluationEvents, type EventHandler, ObservabilityHooks, observabilityHooks, pipelineToSpanAttributes, type SpanAttributes, scorerToSpanAttributes, } from "./observabilityHooks.js";
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Hooks Index
|
|
3
|
+
* Export all observability hooks
|
|
4
|
+
*/
|
|
5
|
+
export { createLangfuseAdapter, createMockLangfuseClient, LangfuseAdapter, startLangfuseAdapter, } from "./langfuseAdapter.js";
|
|
6
|
+
export { createConsoleLoggerHook, createMetricsCollectorHook, ObservabilityHooks, observabilityHooks, pipelineToSpanAttributes, scorerToSpanAttributes, } from "./observabilityHooks.js";
|