@juspay/neurolink 7.29.2 → 7.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/cli/commands/config.d.ts +83 -83
- package/dist/cli/commands/mcp.js +39 -9
- package/dist/cli/commands/models.js +25 -21
- package/dist/cli/commands/ollama.js +2 -2
- package/dist/cli/factories/commandFactory.d.ts +8 -0
- package/dist/cli/factories/commandFactory.js +65 -65
- package/dist/cli/factories/ollamaCommandFactory.js +3 -1
- package/dist/cli/factories/sagemakerCommandFactory.js +3 -2
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +11 -11
- package/dist/cli/utils/envManager.js +5 -5
- package/dist/cli/utils/ollamaUtils.d.ts +12 -0
- package/dist/cli/utils/ollamaUtils.js +58 -42
- package/dist/config/configManager.js +5 -2
- package/dist/config/conversationMemoryConfig.js +5 -0
- package/dist/core/analytics.d.ts +2 -24
- package/dist/core/analytics.js +12 -17
- package/dist/core/baseProvider.d.ts +30 -1
- package/dist/core/baseProvider.js +180 -198
- package/dist/core/conversationMemoryManager.d.ts +9 -15
- package/dist/core/conversationMemoryManager.js +98 -57
- package/dist/core/dynamicModels.d.ts +4 -4
- package/dist/core/dynamicModels.js +7 -7
- package/dist/core/evaluation.d.ts +9 -9
- package/dist/core/evaluation.js +117 -65
- package/dist/core/evaluationProviders.d.ts +18 -2
- package/dist/core/evaluationProviders.js +15 -13
- package/dist/core/modelConfiguration.d.ts +63 -0
- package/dist/core/modelConfiguration.js +354 -290
- package/dist/core/streamAnalytics.d.ts +10 -5
- package/dist/core/streamAnalytics.js +10 -10
- package/dist/core/types.d.ts +22 -110
- package/dist/core/types.js +13 -0
- package/dist/factories/providerFactory.js +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/lib/config/configManager.js +5 -2
- package/dist/lib/config/conversationMemoryConfig.js +5 -0
- package/dist/lib/core/analytics.d.ts +2 -24
- package/dist/lib/core/analytics.js +12 -17
- package/dist/lib/core/baseProvider.d.ts +30 -1
- package/dist/lib/core/baseProvider.js +180 -198
- package/dist/lib/core/conversationMemoryManager.d.ts +9 -15
- package/dist/lib/core/conversationMemoryManager.js +98 -57
- package/dist/lib/core/dynamicModels.js +7 -7
- package/dist/lib/core/evaluation.d.ts +9 -9
- package/dist/lib/core/evaluation.js +117 -65
- package/dist/lib/core/evaluationProviders.d.ts +18 -2
- package/dist/lib/core/evaluationProviders.js +15 -13
- package/dist/lib/core/modelConfiguration.d.ts +63 -0
- package/dist/lib/core/modelConfiguration.js +354 -290
- package/dist/lib/core/streamAnalytics.d.ts +10 -5
- package/dist/lib/core/streamAnalytics.js +10 -10
- package/dist/lib/core/types.d.ts +22 -110
- package/dist/lib/core/types.js +13 -0
- package/dist/lib/factories/providerFactory.js +1 -1
- package/dist/lib/index.d.ts +2 -1
- package/dist/lib/mcp/externalServerManager.js +15 -6
- package/dist/lib/mcp/factory.js +1 -1
- package/dist/lib/mcp/index.d.ts +1 -1
- package/dist/lib/mcp/index.js +1 -1
- package/dist/lib/mcp/mcpCircuitBreaker.js +5 -1
- package/dist/lib/mcp/mcpClientFactory.js +3 -0
- package/dist/lib/mcp/registry.d.ts +3 -3
- package/dist/lib/mcp/registry.js +3 -3
- package/dist/lib/mcp/servers/aiProviders/aiAnalysisTools.js +5 -5
- package/dist/lib/mcp/servers/aiProviders/aiWorkflowTools.js +6 -6
- package/dist/lib/mcp/servers/utilities/utilityServer.js +1 -1
- package/dist/lib/mcp/toolDiscoveryService.js +8 -2
- package/dist/lib/mcp/toolRegistry.js +4 -4
- package/dist/lib/middleware/builtin/analytics.js +4 -4
- package/dist/lib/middleware/builtin/guardrails.js +2 -2
- package/dist/lib/middleware/registry.js +11 -2
- package/dist/lib/models/modelRegistry.d.ts +1 -1
- package/dist/lib/models/modelRegistry.js +3 -3
- package/dist/lib/models/modelResolver.d.ts +1 -1
- package/dist/lib/models/modelResolver.js +2 -2
- package/dist/lib/neurolink.d.ts +116 -9
- package/dist/lib/neurolink.js +718 -956
- package/dist/lib/providers/amazonSagemaker.d.ts +1 -1
- package/dist/lib/providers/amazonSagemaker.js +12 -3
- package/dist/lib/providers/anthropic.d.ts +1 -1
- package/dist/lib/providers/anthropic.js +7 -6
- package/dist/lib/providers/anthropicBaseProvider.d.ts +1 -1
- package/dist/lib/providers/anthropicBaseProvider.js +4 -3
- package/dist/lib/providers/azureOpenai.d.ts +1 -1
- package/dist/lib/providers/azureOpenai.js +1 -1
- package/dist/lib/providers/googleAiStudio.d.ts +1 -1
- package/dist/lib/providers/googleAiStudio.js +2 -2
- package/dist/lib/providers/googleVertex.d.ts +40 -0
- package/dist/lib/providers/googleVertex.js +330 -274
- package/dist/lib/providers/huggingFace.js +1 -1
- package/dist/lib/providers/mistral.d.ts +1 -1
- package/dist/lib/providers/mistral.js +2 -2
- package/dist/lib/providers/ollama.d.ts +4 -0
- package/dist/lib/providers/ollama.js +38 -18
- package/dist/lib/providers/openAI.d.ts +1 -1
- package/dist/lib/providers/openAI.js +2 -2
- package/dist/lib/providers/sagemaker/adaptive-semaphore.js +7 -4
- package/dist/lib/providers/sagemaker/client.js +13 -3
- package/dist/lib/providers/sagemaker/config.js +5 -1
- package/dist/lib/providers/sagemaker/detection.js +19 -9
- package/dist/lib/providers/sagemaker/errors.d.ts +8 -1
- package/dist/lib/providers/sagemaker/errors.js +103 -20
- package/dist/lib/providers/sagemaker/language-model.d.ts +3 -3
- package/dist/lib/providers/sagemaker/language-model.js +4 -4
- package/dist/lib/providers/sagemaker/parsers.js +14 -6
- package/dist/lib/providers/sagemaker/streaming.js +14 -3
- package/dist/lib/providers/sagemaker/types.d.ts +1 -1
- package/dist/lib/proxy/awsProxyIntegration.js +1 -1
- package/dist/lib/sdk/toolRegistration.d.ts +1 -1
- package/dist/lib/types/cli.d.ts +80 -8
- package/dist/lib/types/contextTypes.js +2 -2
- package/dist/lib/types/conversationTypes.d.ts +10 -0
- package/dist/lib/types/generateTypes.d.ts +2 -5
- package/dist/lib/types/providers.d.ts +81 -19
- package/dist/lib/types/providers.js +6 -6
- package/dist/lib/types/streamTypes.d.ts +4 -6
- package/dist/lib/types/typeAliases.d.ts +1 -1
- package/dist/lib/utils/analyticsUtils.d.ts +33 -0
- package/dist/lib/utils/analyticsUtils.js +76 -0
- package/dist/lib/utils/conversationMemoryUtils.d.ts +1 -2
- package/dist/lib/utils/conversationMemoryUtils.js +6 -7
- package/dist/lib/utils/errorHandling.js +4 -1
- package/dist/lib/utils/evaluationUtils.d.ts +27 -0
- package/dist/lib/utils/evaluationUtils.js +131 -0
- package/dist/lib/utils/optionsUtils.js +10 -1
- package/dist/lib/utils/performance.d.ts +1 -1
- package/dist/lib/utils/performance.js +15 -3
- package/dist/lib/utils/providerHealth.d.ts +48 -0
- package/dist/lib/utils/providerHealth.js +199 -254
- package/dist/lib/utils/providerUtils.js +2 -2
- package/dist/lib/utils/timeout.js +8 -3
- package/dist/mcp/externalServerManager.js +15 -6
- package/dist/mcp/factory.js +1 -1
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +1 -1
- package/dist/mcp/mcpCircuitBreaker.js +5 -1
- package/dist/mcp/mcpClientFactory.js +3 -0
- package/dist/mcp/registry.d.ts +3 -3
- package/dist/mcp/registry.js +3 -3
- package/dist/mcp/servers/aiProviders/aiAnalysisTools.js +5 -5
- package/dist/mcp/servers/aiProviders/aiWorkflowTools.js +6 -6
- package/dist/mcp/servers/utilities/utilityServer.js +1 -1
- package/dist/mcp/toolDiscoveryService.js +8 -2
- package/dist/mcp/toolRegistry.js +4 -4
- package/dist/middleware/builtin/analytics.js +4 -4
- package/dist/middleware/builtin/guardrails.js +2 -2
- package/dist/middleware/registry.js +11 -2
- package/dist/models/modelRegistry.d.ts +1 -1
- package/dist/models/modelRegistry.js +3 -3
- package/dist/models/modelResolver.d.ts +1 -1
- package/dist/models/modelResolver.js +2 -2
- package/dist/neurolink.d.ts +116 -9
- package/dist/neurolink.js +718 -956
- package/dist/providers/amazonSagemaker.d.ts +1 -1
- package/dist/providers/amazonSagemaker.js +12 -3
- package/dist/providers/anthropic.d.ts +1 -1
- package/dist/providers/anthropic.js +7 -6
- package/dist/providers/anthropicBaseProvider.d.ts +1 -1
- package/dist/providers/anthropicBaseProvider.js +4 -3
- package/dist/providers/azureOpenai.d.ts +1 -1
- package/dist/providers/azureOpenai.js +1 -1
- package/dist/providers/googleAiStudio.d.ts +1 -1
- package/dist/providers/googleAiStudio.js +2 -2
- package/dist/providers/googleVertex.d.ts +40 -0
- package/dist/providers/googleVertex.js +330 -274
- package/dist/providers/huggingFace.js +1 -1
- package/dist/providers/mistral.d.ts +1 -1
- package/dist/providers/mistral.js +2 -2
- package/dist/providers/ollama.d.ts +4 -0
- package/dist/providers/ollama.js +38 -18
- package/dist/providers/openAI.d.ts +1 -1
- package/dist/providers/openAI.js +2 -2
- package/dist/providers/sagemaker/adaptive-semaphore.js +7 -4
- package/dist/providers/sagemaker/client.js +13 -3
- package/dist/providers/sagemaker/config.js +5 -1
- package/dist/providers/sagemaker/detection.js +19 -9
- package/dist/providers/sagemaker/errors.d.ts +8 -1
- package/dist/providers/sagemaker/errors.js +103 -20
- package/dist/providers/sagemaker/language-model.d.ts +3 -3
- package/dist/providers/sagemaker/language-model.js +4 -4
- package/dist/providers/sagemaker/parsers.js +14 -6
- package/dist/providers/sagemaker/streaming.js +14 -3
- package/dist/providers/sagemaker/types.d.ts +1 -1
- package/dist/proxy/awsProxyIntegration.js +1 -1
- package/dist/sdk/toolRegistration.d.ts +1 -1
- package/dist/types/cli.d.ts +80 -8
- package/dist/types/contextTypes.js +2 -2
- package/dist/types/conversationTypes.d.ts +10 -0
- package/dist/types/generateTypes.d.ts +2 -5
- package/dist/types/providers.d.ts +81 -19
- package/dist/types/providers.js +6 -6
- package/dist/types/streamTypes.d.ts +4 -6
- package/dist/types/typeAliases.d.ts +1 -1
- package/dist/utils/analyticsUtils.d.ts +33 -0
- package/dist/utils/analyticsUtils.js +76 -0
- package/dist/utils/conversationMemoryUtils.d.ts +1 -2
- package/dist/utils/conversationMemoryUtils.js +6 -7
- package/dist/utils/errorHandling.js +4 -1
- package/dist/utils/evaluationUtils.d.ts +27 -0
- package/dist/utils/evaluationUtils.js +131 -0
- package/dist/utils/optionsUtils.js +10 -1
- package/dist/utils/performance.d.ts +1 -1
- package/dist/utils/performance.js +15 -3
- package/dist/utils/providerHealth.d.ts +48 -0
- package/dist/utils/providerHealth.js +199 -254
- package/dist/utils/providerUtils.js +2 -2
- package/dist/utils/timeout.js +8 -3
- package/package.json +1 -1
- package/dist/context/ContextManager.d.ts +0 -28
- package/dist/context/ContextManager.js +0 -113
- package/dist/context/config.d.ts +0 -5
- package/dist/context/config.js +0 -42
- package/dist/context/types.d.ts +0 -20
- package/dist/context/types.js +0 -1
- package/dist/context/utils.d.ts +0 -7
- package/dist/context/utils.js +0 -8
- package/dist/lib/context/ContextManager.d.ts +0 -28
- package/dist/lib/context/ContextManager.js +0 -113
- package/dist/lib/context/config.d.ts +0 -5
- package/dist/lib/context/config.js +0 -42
- package/dist/lib/context/types.d.ts +0 -20
- package/dist/lib/context/types.js +0 -1
- package/dist/lib/context/utils.d.ts +0 -7
- package/dist/lib/context/utils.js +0 -8
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
* Handles in-memory conversation storage, session management, and context injection
|
|
4
4
|
*/
|
|
5
5
|
import { ConversationMemoryError } from "../types/conversationTypes.js";
|
|
6
|
-
import { DEFAULT_MAX_TURNS_PER_SESSION, DEFAULT_MAX_SESSIONS, MESSAGES_PER_TURN
|
|
6
|
+
import { DEFAULT_MAX_TURNS_PER_SESSION, DEFAULT_MAX_SESSIONS, MESSAGES_PER_TURN } from "../config/conversationMemoryConfig.js";
|
|
7
7
|
import { logger } from "../utils/logger.js";
|
|
8
|
+
import { NeuroLink } from "../neurolink.js";
|
|
8
9
|
export class ConversationMemoryManager {
|
|
9
10
|
sessions = new Map();
|
|
10
11
|
config;
|
|
11
12
|
isInitialized = false;
|
|
12
13
|
constructor(config) {
|
|
13
|
-
// Trust that config is already complete from applyConversationMemoryDefaults()
|
|
14
14
|
this.config = config;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
@@ -48,21 +48,19 @@ export class ConversationMemoryManager {
|
|
|
48
48
|
// ULTRA-OPTIMIZED: Direct message storage - no intermediate objects
|
|
49
49
|
session.messages.push({ role: "user", content: userMessage }, { role: "assistant", content: aiResponse });
|
|
50
50
|
session.lastActivity = Date.now();
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
51
|
+
if (this.config.enableSummarization) {
|
|
52
|
+
const currentTurnCount = session.messages.length / MESSAGES_PER_TURN;
|
|
53
|
+
if (currentTurnCount > (this.config.summarizationThresholdTurns || 20)) {
|
|
54
|
+
await this._summarizeSession(session);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
const maxMessages = (this.config.maxTurnsPerSession || DEFAULT_MAX_TURNS_PER_SESSION) * MESSAGES_PER_TURN;
|
|
59
|
+
if (session.messages.length > maxMessages) {
|
|
60
|
+
session.messages = session.messages.slice(-maxMessages);
|
|
61
|
+
}
|
|
56
62
|
}
|
|
57
|
-
// Enforce global session limit
|
|
58
63
|
this.enforceSessionLimit();
|
|
59
|
-
logger.debug("Conversation turn stored", {
|
|
60
|
-
sessionId,
|
|
61
|
-
messageCount: session.messages.length,
|
|
62
|
-
turnCount: session.messages.length / MESSAGES_PER_TURN, // Each turn = MESSAGES_PER_TURN messages
|
|
63
|
-
userMessageLength: userMessage.length,
|
|
64
|
-
aiResponseLength: aiResponse.length,
|
|
65
|
-
});
|
|
66
64
|
}
|
|
67
65
|
catch (error) {
|
|
68
66
|
throw new ConversationMemoryError(`Failed to store conversation turn for session ${sessionId}`, "STORAGE_ERROR", {
|
|
@@ -77,48 +75,74 @@ export class ConversationMemoryManager {
|
|
|
77
75
|
*/
|
|
78
76
|
buildContextMessages(sessionId) {
|
|
79
77
|
const session = this.sessions.get(sessionId);
|
|
80
|
-
|
|
81
|
-
return [];
|
|
82
|
-
}
|
|
83
|
-
// ULTRA-OPTIMIZED: Direct return - no processing needed!
|
|
84
|
-
return session.messages;
|
|
78
|
+
return session ? session.messages : [];
|
|
85
79
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
async getStats() {
|
|
91
|
-
await this.ensureInitialized();
|
|
92
|
-
const sessions = Array.from(this.sessions.values());
|
|
93
|
-
const totalTurns = sessions.reduce((sum, session) => sum + session.messages.length / MESSAGES_PER_TURN, 0);
|
|
80
|
+
getSession(sessionId) {
|
|
81
|
+
return this.sessions.get(sessionId);
|
|
82
|
+
}
|
|
83
|
+
createSummarySystemMessage(content) {
|
|
94
84
|
return {
|
|
95
|
-
|
|
96
|
-
|
|
85
|
+
role: "system",
|
|
86
|
+
content: `Summary of previous conversation turns:\n\n${content}`,
|
|
97
87
|
};
|
|
98
88
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
89
|
+
async _summarizeSession(session) {
|
|
90
|
+
logger.info(`[ConversationMemory] Summarizing session ${session.sessionId}...`);
|
|
91
|
+
const targetTurns = this.config.summarizationTargetTurns || 10;
|
|
92
|
+
const splitIndex = Math.max(0, session.messages.length - targetTurns * MESSAGES_PER_TURN);
|
|
93
|
+
const messagesToSummarize = session.messages.slice(0, splitIndex);
|
|
94
|
+
const recentMessages = session.messages.slice(splitIndex);
|
|
95
|
+
if (messagesToSummarize.length === 0) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const summarizationPrompt = this._createSummarizationPrompt(messagesToSummarize);
|
|
99
|
+
const summarizer = new NeuroLink({ conversationMemory: { enabled: false } });
|
|
100
|
+
try {
|
|
101
|
+
const providerName = this.config.summarizationProvider;
|
|
102
|
+
// Map provider names to correct format
|
|
103
|
+
let mappedProvider = providerName;
|
|
104
|
+
if (providerName === 'vertex') {
|
|
105
|
+
mappedProvider = 'googlevertex';
|
|
106
|
+
}
|
|
107
|
+
if (!mappedProvider) {
|
|
108
|
+
logger.error(`[ConversationMemory] Missing summarization provider`);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
logger.debug(`[ConversationMemory] Using provider: ${mappedProvider} for summarization`);
|
|
112
|
+
const summaryResult = await summarizer.generate({
|
|
113
|
+
input: { text: summarizationPrompt },
|
|
114
|
+
provider: mappedProvider,
|
|
115
|
+
model: this.config.summarizationModel,
|
|
116
|
+
disableTools: true,
|
|
117
|
+
});
|
|
118
|
+
if (summaryResult.content) {
|
|
119
|
+
session.messages = [
|
|
120
|
+
this.createSummarySystemMessage(summaryResult.content),
|
|
121
|
+
...recentMessages
|
|
122
|
+
];
|
|
123
|
+
logger.info(`[ConversationMemory] Summarization complete for session ${session.sessionId}.`);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
logger.warn(`[ConversationMemory] Summarization failed for session ${session.sessionId}. History not modified.`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
logger.error(`[ConversationMemory] Error during summarization for session ${session.sessionId}`, { error });
|
|
106
131
|
}
|
|
107
|
-
// Remove from memory
|
|
108
|
-
this.sessions.delete(sessionId);
|
|
109
|
-
logger.info("Session cleared", { sessionId });
|
|
110
|
-
return true;
|
|
111
132
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
133
|
+
_createSummarizationPrompt(history) {
|
|
134
|
+
const formattedHistory = history.map(msg => `${msg.role}: ${msg.content}`).join('\n\n');
|
|
135
|
+
return `
|
|
136
|
+
You are a context summarization AI. Your task is to condense the following conversation history for another AI assistant.
|
|
137
|
+
The summary must be a concise, third-person narrative that retains all critical information, including key entities, technical details, decisions made, and any specific dates or times mentioned.
|
|
138
|
+
Ensure the summary flows logically and is ready to be used as context for the next turn in the conversation.
|
|
139
|
+
|
|
140
|
+
Conversation History to Summarize:
|
|
141
|
+
---
|
|
142
|
+
${formattedHistory}
|
|
143
|
+
---
|
|
144
|
+
`.trim();
|
|
120
145
|
}
|
|
121
|
-
// Private methods
|
|
122
146
|
async ensureInitialized() {
|
|
123
147
|
if (!this.isInitialized) {
|
|
124
148
|
await this.initialize();
|
|
@@ -138,16 +162,33 @@ export class ConversationMemoryManager {
|
|
|
138
162
|
if (this.sessions.size <= maxSessions) {
|
|
139
163
|
return;
|
|
140
164
|
}
|
|
141
|
-
// Sort sessions by last activity (oldest first)
|
|
142
165
|
const sessions = Array.from(this.sessions.entries()).sort(([, a], [, b]) => a.lastActivity - b.lastActivity);
|
|
143
|
-
|
|
144
|
-
const sessionsToRemove = sessions.slice(0, sessions.length - maxSessions);
|
|
166
|
+
const sessionsToRemove = sessions.slice(0, this.sessions.size - maxSessions);
|
|
145
167
|
for (const [sessionId] of sessionsToRemove) {
|
|
146
168
|
this.sessions.delete(sessionId);
|
|
147
169
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
170
|
+
}
|
|
171
|
+
async getStats() {
|
|
172
|
+
await this.ensureInitialized();
|
|
173
|
+
const sessions = Array.from(this.sessions.values());
|
|
174
|
+
const totalTurns = sessions.reduce((sum, session) => sum + session.messages.length / MESSAGES_PER_TURN, 0);
|
|
175
|
+
return {
|
|
176
|
+
totalSessions: sessions.length,
|
|
177
|
+
totalTurns,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
async clearSession(sessionId) {
|
|
181
|
+
const session = this.sessions.get(sessionId);
|
|
182
|
+
if (!session) {
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
this.sessions.delete(sessionId);
|
|
186
|
+
logger.info("Session cleared", { sessionId });
|
|
187
|
+
return true;
|
|
188
|
+
}
|
|
189
|
+
async clearAllSessions() {
|
|
190
|
+
const sessionIds = Array.from(this.sessions.keys());
|
|
191
|
+
this.sessions.clear();
|
|
192
|
+
logger.info("All sessions cleared", { clearedCount: sessionIds.length });
|
|
152
193
|
}
|
|
153
194
|
}
|
|
@@ -88,6 +88,8 @@ declare const ModelRegistrySchema: z.ZodObject<{
|
|
|
88
88
|
aliases: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
89
89
|
defaults: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
90
90
|
}, "strip", z.ZodTypeAny, {
|
|
91
|
+
version: string;
|
|
92
|
+
lastUpdated: string;
|
|
91
93
|
models: Record<string, Record<string, {
|
|
92
94
|
capabilities: string[];
|
|
93
95
|
id: string;
|
|
@@ -100,11 +102,11 @@ declare const ModelRegistrySchema: z.ZodObject<{
|
|
|
100
102
|
contextWindow: number;
|
|
101
103
|
releaseDate: string;
|
|
102
104
|
}>>;
|
|
103
|
-
version: string;
|
|
104
|
-
lastUpdated: string;
|
|
105
105
|
defaults?: Record<string, string> | undefined;
|
|
106
106
|
aliases?: Record<string, string> | undefined;
|
|
107
107
|
}, {
|
|
108
|
+
version: string;
|
|
109
|
+
lastUpdated: string;
|
|
108
110
|
models: Record<string, Record<string, {
|
|
109
111
|
capabilities: string[];
|
|
110
112
|
id: string;
|
|
@@ -117,8 +119,6 @@ declare const ModelRegistrySchema: z.ZodObject<{
|
|
|
117
119
|
contextWindow: number;
|
|
118
120
|
releaseDate: string;
|
|
119
121
|
}>>;
|
|
120
|
-
version: string;
|
|
121
|
-
lastUpdated: string;
|
|
122
122
|
defaults?: Record<string, string> | undefined;
|
|
123
123
|
aliases?: Record<string, string> | undefined;
|
|
124
124
|
}>;
|
|
@@ -100,12 +100,6 @@ export class DynamicModelProvider {
|
|
|
100
100
|
// Setup timeout and abort controller
|
|
101
101
|
const controller = new AbortController();
|
|
102
102
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
103
|
-
// Load from URL
|
|
104
|
-
const response = await fetch(source, {
|
|
105
|
-
headers: {
|
|
106
|
-
"User-Agent": "NeuroLink/1.0 (+https://github.com/juspay/neurolink)",
|
|
107
|
-
},
|
|
108
|
-
});
|
|
109
103
|
try {
|
|
110
104
|
// Add health check for localhost before attempting full request
|
|
111
105
|
if (source.includes("localhost") || source.includes("127.0.0.1")) {
|
|
@@ -113,7 +107,7 @@ export class DynamicModelProvider {
|
|
|
113
107
|
}
|
|
114
108
|
const response = await fetch(source, {
|
|
115
109
|
headers: {
|
|
116
|
-
"User-Agent": "NeuroLink/1.0 (+https://github.com/
|
|
110
|
+
"User-Agent": "NeuroLink/1.0 (+https://github.com/juspay/neurolink)",
|
|
117
111
|
Accept: "application/json",
|
|
118
112
|
"Cache-Control": "no-cache",
|
|
119
113
|
},
|
|
@@ -242,6 +236,9 @@ export class DynamicModelProvider {
|
|
|
242
236
|
searchByCapability(capability, options = {}) {
|
|
243
237
|
this.ensureInitialized();
|
|
244
238
|
const results = [];
|
|
239
|
+
if (!this.modelRegistry) {
|
|
240
|
+
return results;
|
|
241
|
+
}
|
|
245
242
|
for (const [providerName, models] of Object.entries(this.modelRegistry.models)) {
|
|
246
243
|
if (options.provider && providerName !== options.provider) {
|
|
247
244
|
continue;
|
|
@@ -301,6 +298,9 @@ export class DynamicModelProvider {
|
|
|
301
298
|
getAllModels() {
|
|
302
299
|
this.ensureInitialized();
|
|
303
300
|
const results = [];
|
|
301
|
+
if (!this.modelRegistry) {
|
|
302
|
+
return results;
|
|
303
|
+
}
|
|
304
304
|
for (const [providerName, models] of Object.entries(this.modelRegistry.models)) {
|
|
305
305
|
for (const [modelName, modelConfig] of Object.entries(models)) {
|
|
306
306
|
results.push({
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* NeuroLink
|
|
2
|
+
* NeuroLink Evaluation System
|
|
3
3
|
*/
|
|
4
|
-
import type { EvaluationData } from "
|
|
5
|
-
export interface
|
|
4
|
+
import type { EvaluationData } from "../index.js";
|
|
5
|
+
export interface EvaluationResult extends EvaluationData {
|
|
6
6
|
domainAlignment?: number;
|
|
7
7
|
terminologyAccuracy?: number;
|
|
8
8
|
toolEffectiveness?: number;
|
|
@@ -20,7 +20,7 @@ export interface UnifiedEvaluationResult extends EvaluationData {
|
|
|
20
20
|
alertSeverity: "low" | "medium" | "high" | "none";
|
|
21
21
|
reasoning: string;
|
|
22
22
|
}
|
|
23
|
-
export interface
|
|
23
|
+
export interface EvaluationContext {
|
|
24
24
|
userQuery: string;
|
|
25
25
|
aiResponse: string;
|
|
26
26
|
context?: Record<string, unknown>;
|
|
@@ -43,14 +43,14 @@ export interface UnifiedEvaluationContext {
|
|
|
43
43
|
/**
|
|
44
44
|
* Get default evaluation when evaluation fails
|
|
45
45
|
*/
|
|
46
|
-
declare function
|
|
46
|
+
declare function getDefaultEvaluation(reason: string, evaluationTime: number, context: EvaluationContext): EvaluationResult;
|
|
47
47
|
/**
|
|
48
48
|
* Parse unified evaluation result from text response
|
|
49
49
|
*/
|
|
50
|
-
declare function
|
|
50
|
+
declare function parseEvaluationResult(response: string, context: EvaluationContext): Partial<EvaluationResult>;
|
|
51
51
|
/**
|
|
52
52
|
* Main unified evaluation function
|
|
53
53
|
*/
|
|
54
|
-
export declare function
|
|
55
|
-
export declare function evaluateResponse(
|
|
56
|
-
export {
|
|
54
|
+
export declare function generateEvaluation(context: EvaluationContext): Promise<EvaluationResult>;
|
|
55
|
+
export declare function evaluateResponse(context: EvaluationContext): Promise<EvaluationResult>;
|
|
56
|
+
export { getDefaultEvaluation, parseEvaluationResult };
|
package/dist/core/evaluation.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* NeuroLink
|
|
2
|
+
* NeuroLink Evaluation System
|
|
3
3
|
*/
|
|
4
4
|
import { logger } from "../utils/logger.js";
|
|
5
5
|
import { AIProviderFactory } from "./factory.js";
|
|
6
6
|
import { z } from "zod";
|
|
7
7
|
import { ProviderRegistry } from "../factories/providerRegistry.js";
|
|
8
8
|
import { modelConfig } from "./modelConfiguration.js";
|
|
9
|
+
import { normalizeEvaluationData } from "../utils/evaluationUtils.js";
|
|
9
10
|
// Zod schema for validation
|
|
10
|
-
const
|
|
11
|
+
const EvaluationSchema = z.object({
|
|
11
12
|
relevance: z.number().min(1).max(10),
|
|
12
13
|
accuracy: z.number().min(1).max(10),
|
|
13
14
|
completeness: z.number().min(1).max(10),
|
|
@@ -19,8 +20,8 @@ const UnifiedEvaluationSchema = z.object({
|
|
|
19
20
|
/**
|
|
20
21
|
* Get default evaluation when evaluation fails
|
|
21
22
|
*/
|
|
22
|
-
function
|
|
23
|
-
const functionTag = "
|
|
23
|
+
function getDefaultEvaluation(reason, evaluationTime, context) {
|
|
24
|
+
const functionTag = "getDefaultEvaluation";
|
|
24
25
|
logger.debug(`[${functionTag}] Creating default evaluation`, {
|
|
25
26
|
reason,
|
|
26
27
|
evaluationTime,
|
|
@@ -61,11 +62,14 @@ function getDefaultUnifiedEvaluation(reason, evaluationTime, context) {
|
|
|
61
62
|
/**
|
|
62
63
|
* Parse unified evaluation result from text response
|
|
63
64
|
*/
|
|
64
|
-
function
|
|
65
|
-
const functionTag = "
|
|
65
|
+
function parseEvaluationResult(response, context) {
|
|
66
|
+
const functionTag = "parseEvaluationResult";
|
|
66
67
|
try {
|
|
67
68
|
logger.debug(`[${functionTag}] Parsing evaluation response`, {
|
|
68
69
|
responseLength: response.length,
|
|
70
|
+
domain: context.primaryDomain,
|
|
71
|
+
hasToolUsage: !!context.toolUsage?.length,
|
|
72
|
+
hasConversationHistory: !!context.conversationHistory?.length,
|
|
69
73
|
});
|
|
70
74
|
// Try JSON parsing first
|
|
71
75
|
const jsonMatch = response.match(/\{[^}]*\}/s);
|
|
@@ -74,8 +78,11 @@ function parseUnifiedEvaluationResult(response, context) {
|
|
|
74
78
|
const parsed = JSON.parse(jsonMatch[0]);
|
|
75
79
|
return parsed;
|
|
76
80
|
}
|
|
77
|
-
catch (
|
|
78
|
-
logger.debug(`[${functionTag}] JSON parsing failed, trying regex
|
|
81
|
+
catch (jsonError) {
|
|
82
|
+
logger.debug(`[${functionTag}] JSON parsing failed, trying regex`, {
|
|
83
|
+
error: jsonError instanceof Error ? jsonError.message : String(jsonError),
|
|
84
|
+
jsonContent: jsonMatch[0].substring(0, 100), // First 100 chars for debugging
|
|
85
|
+
});
|
|
79
86
|
}
|
|
80
87
|
}
|
|
81
88
|
// Fallback to regex parsing
|
|
@@ -115,14 +122,49 @@ function parseUnifiedEvaluationResult(response, context) {
|
|
|
115
122
|
}
|
|
116
123
|
}
|
|
117
124
|
}
|
|
118
|
-
// Ensure minimum valid scores
|
|
119
|
-
|
|
125
|
+
// Ensure minimum valid scores and validate with schema
|
|
126
|
+
// Use context to enhance evaluation data
|
|
127
|
+
const evaluationData = {
|
|
120
128
|
relevance: result.relevance || 1,
|
|
121
129
|
accuracy: result.accuracy || 1,
|
|
122
130
|
completeness: result.completeness || 1,
|
|
123
131
|
overall: result.overall || 1,
|
|
124
|
-
|
|
132
|
+
domainAlignment: result.domainAlignment || (context.primaryDomain ? 5 : undefined), // Default to 5 if domain-specific
|
|
133
|
+
terminologyAccuracy: result.terminologyAccuracy || (context.primaryDomain ? 5 : undefined),
|
|
134
|
+
toolEffectiveness: result.toolEffectiveness || (context.toolUsage?.length ? 5 : undefined), // Default to 5 if tools were used
|
|
125
135
|
};
|
|
136
|
+
// Validate against schema
|
|
137
|
+
try {
|
|
138
|
+
const validated = EvaluationSchema.parse(evaluationData);
|
|
139
|
+
// Enhance reasoning with context information
|
|
140
|
+
let enhancedReasoning = result.reasoning || "No detailed reasoning provided";
|
|
141
|
+
if (context.primaryDomain) {
|
|
142
|
+
enhancedReasoning += ` (Domain: ${context.primaryDomain})`;
|
|
143
|
+
}
|
|
144
|
+
if (context.toolUsage?.length) {
|
|
145
|
+
enhancedReasoning += ` (Tools used: ${context.toolUsage.map((t) => t.toolName).join(", ")})`;
|
|
146
|
+
}
|
|
147
|
+
if (context.conversationHistory?.length) {
|
|
148
|
+
enhancedReasoning += ` (Conversation turns: ${context.conversationHistory.length})`;
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
...validated,
|
|
152
|
+
reasoning: enhancedReasoning,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
catch (validationError) {
|
|
156
|
+
logger.warn(`[${functionTag}] Schema validation failed, using fallback`, {
|
|
157
|
+
validationError,
|
|
158
|
+
originalData: evaluationData,
|
|
159
|
+
});
|
|
160
|
+
return {
|
|
161
|
+
relevance: Math.max(1, Math.min(10, result.relevance || 1)),
|
|
162
|
+
accuracy: Math.max(1, Math.min(10, result.accuracy || 1)),
|
|
163
|
+
completeness: Math.max(1, Math.min(10, result.completeness || 1)),
|
|
164
|
+
overall: Math.max(1, Math.min(10, result.overall || 1)),
|
|
165
|
+
reasoning: result.reasoning || "No detailed reasoning provided",
|
|
166
|
+
};
|
|
167
|
+
}
|
|
126
168
|
}
|
|
127
169
|
catch (error) {
|
|
128
170
|
logger.error(`[${functionTag}] Failed to parse evaluation result`, {
|
|
@@ -140,8 +182,8 @@ function parseUnifiedEvaluationResult(response, context) {
|
|
|
140
182
|
/**
|
|
141
183
|
* Main unified evaluation function
|
|
142
184
|
*/
|
|
143
|
-
export async function
|
|
144
|
-
const functionTag = "
|
|
185
|
+
export async function generateEvaluation(context) {
|
|
186
|
+
const functionTag = "generateEvaluation";
|
|
145
187
|
const startTime = Date.now();
|
|
146
188
|
logger.debug(`[${functionTag}] Starting evaluation`, {
|
|
147
189
|
hasUserQuery: !!context.userQuery,
|
|
@@ -161,7 +203,7 @@ export async function generateUnifiedEvaluation(context) {
|
|
|
161
203
|
const provider = await AIProviderFactory.createProvider(evaluationProvider, evaluationModel);
|
|
162
204
|
if (!provider) {
|
|
163
205
|
logger.debug(`[${functionTag}] No evaluation provider available, returning defaults`);
|
|
164
|
-
return
|
|
206
|
+
return getDefaultEvaluation("no-provider", Date.now() - startTime, context);
|
|
165
207
|
}
|
|
166
208
|
// Create evaluation prompt
|
|
167
209
|
const prompt = `
|
|
@@ -187,68 +229,78 @@ Reasoning: [Provide a detailed explanation of your evaluation, explaining why yo
|
|
|
187
229
|
const result = await provider.generate(prompt);
|
|
188
230
|
if (!result) {
|
|
189
231
|
logger.debug(`[${functionTag}] No response from provider`);
|
|
190
|
-
return
|
|
232
|
+
return getDefaultEvaluation("no-response", Date.now() - startTime, context);
|
|
191
233
|
}
|
|
192
234
|
// Extract text from result
|
|
193
235
|
const response = typeof result === "string" ? result : result?.content || String(result);
|
|
194
236
|
// Parse evaluation result
|
|
195
|
-
const parsed =
|
|
196
|
-
// Validate and enhance result
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
fallbackUsed: false,
|
|
206
|
-
costEstimate: 0.001, // Rough estimate
|
|
207
|
-
},
|
|
237
|
+
const parsed = parseEvaluationResult(response, context);
|
|
238
|
+
// Validate and enhance result using schema
|
|
239
|
+
const baseResult = {
|
|
240
|
+
relevance: parsed.relevance || 1,
|
|
241
|
+
accuracy: parsed.accuracy || 1,
|
|
242
|
+
completeness: parsed.completeness || 1,
|
|
243
|
+
overall: parsed.overall || 1,
|
|
244
|
+
domainAlignment: parsed.domainAlignment,
|
|
245
|
+
terminologyAccuracy: parsed.terminologyAccuracy,
|
|
246
|
+
toolEffectiveness: parsed.toolEffectiveness,
|
|
208
247
|
};
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
248
|
+
// Validate against schema before finalizing
|
|
249
|
+
try {
|
|
250
|
+
const validatedScores = EvaluationSchema.parse(baseResult);
|
|
251
|
+
const validatedResult = normalizeEvaluationData({
|
|
252
|
+
...parsed,
|
|
253
|
+
...validatedScores,
|
|
254
|
+
evaluationModel: `${evaluationProvider}/${evaluationModel}`,
|
|
255
|
+
evaluationTime: Date.now() - startTime,
|
|
256
|
+
evaluationProvider,
|
|
257
|
+
evaluationAttempt: 1,
|
|
258
|
+
evaluationConfig: {
|
|
259
|
+
mode: "standard",
|
|
260
|
+
fallbackUsed: false,
|
|
261
|
+
costEstimate: 0.001, // Rough estimate
|
|
262
|
+
},
|
|
263
|
+
});
|
|
264
|
+
logger.debug(`[${functionTag}] Schema validation passed`, {
|
|
265
|
+
validatedScores,
|
|
266
|
+
});
|
|
267
|
+
return validatedResult;
|
|
268
|
+
}
|
|
269
|
+
catch (validationError) {
|
|
270
|
+
logger.warn(`[${functionTag}] Schema validation failed in main evaluation`, {
|
|
271
|
+
validationError,
|
|
272
|
+
baseResult,
|
|
273
|
+
});
|
|
274
|
+
// Fallback with clamped values using normalizer
|
|
275
|
+
const validatedResult = normalizeEvaluationData({
|
|
276
|
+
...parsed,
|
|
277
|
+
relevance: parsed.relevance || 1,
|
|
278
|
+
accuracy: parsed.accuracy || 1,
|
|
279
|
+
completeness: parsed.completeness || 1,
|
|
280
|
+
overall: parsed.overall || 1,
|
|
281
|
+
evaluationModel: `${evaluationProvider}/${evaluationModel}`,
|
|
282
|
+
evaluationTime: Date.now() - startTime,
|
|
283
|
+
evaluationProvider,
|
|
284
|
+
evaluationAttempt: 1,
|
|
285
|
+
evaluationConfig: {
|
|
286
|
+
mode: "standard",
|
|
287
|
+
fallbackUsed: false,
|
|
288
|
+
costEstimate: 0.001, // Rough estimate
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
return validatedResult;
|
|
292
|
+
}
|
|
217
293
|
}
|
|
218
294
|
catch (error) {
|
|
219
295
|
logger.error(`[${functionTag}] Evaluation failed`, {
|
|
220
296
|
error: error instanceof Error ? error.message : String(error),
|
|
221
297
|
});
|
|
222
|
-
return
|
|
298
|
+
return getDefaultEvaluation(error instanceof Error ? error.message : "unknown-error", Date.now() - startTime, context);
|
|
223
299
|
}
|
|
224
300
|
}
|
|
225
|
-
//
|
|
226
|
-
export async function evaluateResponse(
|
|
227
|
-
|
|
228
|
-
let aiResponse;
|
|
229
|
-
let context;
|
|
230
|
-
if (typeof responseOrContext === "string") {
|
|
231
|
-
// Normal call: evaluateResponse(response, context, ...)
|
|
232
|
-
aiResponse = responseOrContext;
|
|
233
|
-
context = contextOrUserQuery;
|
|
234
|
-
}
|
|
235
|
-
else {
|
|
236
|
-
// Provider call pattern: evaluateResponse(contextObject, userQuery, ...)
|
|
237
|
-
context = responseOrContext;
|
|
238
|
-
aiResponse =
|
|
239
|
-
context?.aiResponse ||
|
|
240
|
-
context?.response ||
|
|
241
|
-
String(contextOrUserQuery || "");
|
|
242
|
-
}
|
|
243
|
-
const evalContext = {
|
|
244
|
-
userQuery: (typeof userQuery === "string" ? userQuery : "") ||
|
|
245
|
-
context?.userQuery ||
|
|
246
|
-
(typeof contextOrUserQuery === "string" ? contextOrUserQuery : "") ||
|
|
247
|
-
"Generated response",
|
|
248
|
-
aiResponse,
|
|
249
|
-
context: context,
|
|
250
|
-
};
|
|
251
|
-
return generateUnifiedEvaluation(evalContext);
|
|
301
|
+
// Simplified evaluation function
|
|
302
|
+
export async function evaluateResponse(context) {
|
|
303
|
+
return generateEvaluation(context);
|
|
252
304
|
}
|
|
253
305
|
// Export additional utilities
|
|
254
|
-
export {
|
|
306
|
+
export { getDefaultEvaluation, parseEvaluationResult };
|
|
@@ -1,4 +1,19 @@
|
|
|
1
|
-
|
|
1
|
+
interface ProviderModelConfig {
|
|
2
|
+
provider: string;
|
|
3
|
+
models: string[];
|
|
4
|
+
costPerToken?: number | {
|
|
5
|
+
input: number;
|
|
6
|
+
output: number;
|
|
7
|
+
};
|
|
8
|
+
requiresApiKey?: string[];
|
|
9
|
+
performance?: {
|
|
10
|
+
averageLatency?: number;
|
|
11
|
+
reliability?: number;
|
|
12
|
+
speed?: number;
|
|
13
|
+
quality?: number;
|
|
14
|
+
cost?: number;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
2
17
|
/**
|
|
3
18
|
* Dynamic provider configurations for evaluation
|
|
4
19
|
* Now uses configurable system instead of hardcoded values
|
|
@@ -22,7 +37,7 @@ export declare function sortProvidersByPreference(providers: ProviderModelConfig
|
|
|
22
37
|
* Estimate cost for a specific provider and token usage
|
|
23
38
|
* Now uses the configurable system
|
|
24
39
|
*/
|
|
25
|
-
export declare function estimateProviderCost(providerName: string,
|
|
40
|
+
export declare function estimateProviderCost(providerName: string, input: number, output: number): number;
|
|
26
41
|
/**
|
|
27
42
|
* Check if a provider is available (has required API keys)
|
|
28
43
|
* Now uses the configurable system
|
|
@@ -58,3 +73,4 @@ export declare function getProviderPerformanceAnalytics(): Record<string, {
|
|
|
58
73
|
* @param providerName - (Optional) The name of the provider to reset. If omitted, resets all providers.
|
|
59
74
|
*/
|
|
60
75
|
export declare function resetProviderMetrics(providerName?: string): void;
|
|
76
|
+
export {};
|