@rws-framework/ai-tools 3.3.1 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/models/convo/EmbedLoader.ts +8 -5
- package/src/models/prompts/inc/execution-methods-handler.ts +16 -3
- package/src/models/prompts/inc/model-execution-manager.ts +1 -1
- package/src/models/prompts/inc/tool-manager.ts +18 -3
- package/src/services/LangChainEmbeddingService.ts +12 -0
- package/src/services/LangChainRAGService.ts +35 -37
- package/src/services/OpenAIRateLimitingService.ts +24 -9
- package/src/services/OptimizedVectorSearchService.ts +33 -21
- package/src/services/TextChunker.ts +3 -2
- package/src/types/IPrompt.ts +1 -2
- package/src/types/rag.types.ts +1 -1
- package/src/types/search.types.ts +3 -3
package/package.json
CHANGED
|
@@ -187,12 +187,15 @@ class EmbedLoader<LLMChat extends BaseChatModel> {
|
|
|
187
187
|
logConvo(`After the split we have ${splitDocs.length} documents more than the original ${orgDocs.length}.`);
|
|
188
188
|
logConvo(`Average length among ${splitDocs.length} documents (after split) is ${avgCharCountPost} characters.`);
|
|
189
189
|
|
|
190
|
-
|
|
191
|
-
splitDocs.
|
|
190
|
+
// Write files asynchronously to prevent blocking
|
|
191
|
+
await Promise.all(splitDocs.map(async (doc: Document, i: number) => {
|
|
192
192
|
finalDocs.push(doc);
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
193
|
+
try {
|
|
194
|
+
await fs.promises.writeFile(this.debugSplitFile(i), doc.pageContent, 'utf-8');
|
|
195
|
+
} catch (error) {
|
|
196
|
+
console.warn(`Failed to write debug file ${i}:`, error);
|
|
197
|
+
}
|
|
198
|
+
}));
|
|
196
199
|
}else{
|
|
197
200
|
const splitFiles = fs.readdirSync(splitDir);
|
|
198
201
|
|
|
@@ -22,9 +22,17 @@ export class ExecutionMethodsHandler {
|
|
|
22
22
|
debugVars: any = {},
|
|
23
23
|
tools?: IAITool[]
|
|
24
24
|
): Promise<void> {
|
|
25
|
-
|
|
25
|
+
// Create snapshot of current input to prevent race conditions
|
|
26
|
+
const inputSnapshot = [...promptInstance.getInput()];
|
|
27
|
+
promptInstance.setSentInput(inputSnapshot);
|
|
28
|
+
|
|
26
29
|
const returnedRWS = await executor.promptRequest(promptInstance as any, { intruderPrompt, debugVars, tools });
|
|
27
|
-
|
|
30
|
+
|
|
31
|
+
// Safely ingest output
|
|
32
|
+
const output = returnedRWS.readOutput();
|
|
33
|
+
if (output !== null && output !== undefined) {
|
|
34
|
+
promptInstance.injestOutput(output);
|
|
35
|
+
}
|
|
28
36
|
}
|
|
29
37
|
|
|
30
38
|
async singleRequestWith(
|
|
@@ -34,8 +42,13 @@ export class ExecutionMethodsHandler {
|
|
|
34
42
|
ensureJson: boolean = false,
|
|
35
43
|
tools?: IAITool[]
|
|
36
44
|
): Promise<void> {
|
|
45
|
+
// Create snapshot of current input to prevent race conditions
|
|
46
|
+
const inputSnapshot = [...promptInstance.getInput()];
|
|
47
|
+
|
|
37
48
|
await executor.singlePromptRequest(promptInstance as any, { intruderPrompt, ensureJson, tools });
|
|
38
|
-
|
|
49
|
+
|
|
50
|
+
// Set the snapshot after execution to maintain consistency
|
|
51
|
+
promptInstance.setSentInput(inputSnapshot);
|
|
39
52
|
}
|
|
40
53
|
|
|
41
54
|
async streamWith(
|
|
@@ -22,7 +22,7 @@ export class ModelExecutionManager {
|
|
|
22
22
|
constructor(modelId: string, modelType: string, hyperParameters: IPromptHyperParameters) {
|
|
23
23
|
this.modelId = modelId;
|
|
24
24
|
this.modelType = modelType;
|
|
25
|
-
this.hyperParameters = hyperParameters;
|
|
25
|
+
this.hyperParameters = hyperParameters || { temperature: 0.7, max_tokens: 512 };
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
getModelId(): string {
|
|
@@ -34,14 +34,29 @@ export class ToolManager {
|
|
|
34
34
|
|
|
35
35
|
async callTools<T = unknown, O = unknown>(tools: IToolCall[], moduleRef: ModuleRef, aiToolOptions?: O): Promise<T[]> {
|
|
36
36
|
const results: T[] = [];
|
|
37
|
+
const errors: Error[] = [];
|
|
38
|
+
|
|
37
39
|
for (const tool of tools) {
|
|
38
40
|
if (this.toolHandlers.has(tool.function.name)) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
41
|
+
try {
|
|
42
|
+
const result = await this.callAiTool<T, O>(tool, moduleRef, aiToolOptions);
|
|
43
|
+
if (result) {
|
|
44
|
+
results.push(result);
|
|
45
|
+
}
|
|
46
|
+
} catch (error) {
|
|
47
|
+
console.error(`Tool execution failed for ${tool.function.name}:`, error);
|
|
48
|
+
errors.push(error as Error);
|
|
49
|
+
// Continue with other tools instead of failing completely
|
|
42
50
|
}
|
|
51
|
+
} else {
|
|
52
|
+
console.warn(`No handler found for tool: ${tool.function.name}`);
|
|
43
53
|
}
|
|
44
54
|
}
|
|
55
|
+
|
|
56
|
+
// If all tools failed, throw the first error
|
|
57
|
+
if (results.length === 0 && errors.length > 0) {
|
|
58
|
+
throw errors[0];
|
|
59
|
+
}
|
|
45
60
|
|
|
46
61
|
return results;
|
|
47
62
|
}
|
|
@@ -15,6 +15,7 @@ export class LangChainEmbeddingService {
|
|
|
15
15
|
private chunkConfig: IChunkConfig;
|
|
16
16
|
private isInitialized = false;
|
|
17
17
|
private vectorStore: RWSVectorStore | null = null;
|
|
18
|
+
private static embeddingsPool = new Map<string, Embeddings>(); // Connection pooling
|
|
18
19
|
|
|
19
20
|
constructor(private rateLimitingService: OpenAIRateLimitingService) {}
|
|
20
21
|
|
|
@@ -37,6 +38,14 @@ export class LangChainEmbeddingService {
|
|
|
37
38
|
|
|
38
39
|
|
|
39
40
|
private initializeEmbeddings(): void {
|
|
41
|
+
const poolKey = `${this.config.provider}_${this.config.model}_${this.config.apiKey.slice(-8)}`;
|
|
42
|
+
|
|
43
|
+
// Check connection pool first
|
|
44
|
+
if (LangChainEmbeddingService.embeddingsPool.has(poolKey)) {
|
|
45
|
+
this.embeddings = LangChainEmbeddingService.embeddingsPool.get(poolKey)!;
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
|
|
40
49
|
switch (this.config.provider) {
|
|
41
50
|
case 'cohere':
|
|
42
51
|
this.embeddings = new CohereEmbeddings({
|
|
@@ -58,6 +67,9 @@ export class LangChainEmbeddingService {
|
|
|
58
67
|
default:
|
|
59
68
|
throw new Error(`Unsupported embedding provider: ${this.config.provider}`);
|
|
60
69
|
}
|
|
70
|
+
|
|
71
|
+
// Store in connection pool for reuse
|
|
72
|
+
LangChainEmbeddingService.embeddingsPool.set(poolKey, this.embeddings);
|
|
61
73
|
|
|
62
74
|
if(this.config.rateLimiting){
|
|
63
75
|
const rateLimitingCfg = {...OpenAIRateLimitingService.DEFAULT_CONFIG, ...this.config.rateLimiting};
|
|
@@ -41,8 +41,6 @@ export {
|
|
|
41
41
|
export class LangChainRAGService {
|
|
42
42
|
private config: ILangChainRAGConfig;
|
|
43
43
|
private isInitialized = false;
|
|
44
|
-
private queryEmbeddingCache = new Map<string, number[]>();
|
|
45
|
-
private maxCacheSize = 100;
|
|
46
44
|
private logger?: any; // Optional logger interface
|
|
47
45
|
|
|
48
46
|
constructor(
|
|
@@ -85,11 +83,11 @@ export class LangChainRAGService {
|
|
|
85
83
|
* Index knowledge content for RAG with optimized per-knowledge vector storage
|
|
86
84
|
*/
|
|
87
85
|
async indexKnowledge(
|
|
88
|
-
|
|
86
|
+
fileId: string | number,
|
|
89
87
|
content: string,
|
|
90
88
|
metadata: Record<string, any> = {}
|
|
91
89
|
): Promise<IRAGResponse<{ chunkIds: string[] }>> {
|
|
92
|
-
this.log('log', `[INDEXING] Starting indexKnowledge for
|
|
90
|
+
this.log('log', `[INDEXING] Starting indexKnowledge for fileId: ${fileId}`);
|
|
93
91
|
this.log('debug', `[INDEXING] Content length: ${content.length} characters`);
|
|
94
92
|
|
|
95
93
|
try {
|
|
@@ -97,7 +95,7 @@ export class LangChainRAGService {
|
|
|
97
95
|
|
|
98
96
|
// Chunk the content using the embedding service
|
|
99
97
|
const chunks = await this.embeddingService.chunkText(content);
|
|
100
|
-
this.log('debug', `[INDEXING] Split content into ${chunks.length} chunks for
|
|
98
|
+
this.log('debug', `[INDEXING] Split content into ${chunks.length} chunks for file ${fileId}`);
|
|
101
99
|
|
|
102
100
|
// Generate embeddings for all chunks at once (batch processing for speed)
|
|
103
101
|
const embeddings = await this.embeddingService.embedTexts(chunks);
|
|
@@ -109,17 +107,17 @@ export class LangChainRAGService {
|
|
|
109
107
|
embedding: embeddings[index],
|
|
110
108
|
metadata: {
|
|
111
109
|
...metadata,
|
|
112
|
-
|
|
110
|
+
fileId,
|
|
113
111
|
chunkIndex: index,
|
|
114
|
-
id: `knowledge_${
|
|
112
|
+
id: `knowledge_${fileId}_chunk_${index}`
|
|
115
113
|
}
|
|
116
114
|
}));
|
|
117
115
|
|
|
118
116
|
// Save to per-knowledge vector file
|
|
119
|
-
await this.saveKnowledgeVector(
|
|
117
|
+
await this.saveKnowledgeVector(fileId, chunksWithEmbeddings);
|
|
120
118
|
|
|
121
119
|
const chunkIds = chunksWithEmbeddings.map(chunk => chunk.metadata.id);
|
|
122
|
-
this.log('log', `[INDEXING] Successfully indexed
|
|
120
|
+
this.log('log', `[INDEXING] Successfully indexed file ${fileId} with ${chunkIds.length} chunks using optimized approach`);
|
|
123
121
|
|
|
124
122
|
return {
|
|
125
123
|
success: true,
|
|
@@ -127,7 +125,7 @@ export class LangChainRAGService {
|
|
|
127
125
|
};
|
|
128
126
|
|
|
129
127
|
} catch (error: any) {
|
|
130
|
-
this.log('error', `[INDEXING] Failed to index
|
|
128
|
+
this.log('error', `[INDEXING] Failed to index file ${fileId}:`, error);
|
|
131
129
|
return {
|
|
132
130
|
success: false,
|
|
133
131
|
data: null,
|
|
@@ -146,11 +144,11 @@ export class LangChainRAGService {
|
|
|
146
144
|
try {
|
|
147
145
|
await this.ensureInitialized();
|
|
148
146
|
|
|
149
|
-
const
|
|
150
|
-
console.log('
|
|
147
|
+
const fileIds = request.filter?.fileIds || [];
|
|
148
|
+
console.log('fileIds', fileIds, 'temporaryDocumentSearch:', request.temporaryDocumentSearch);
|
|
151
149
|
|
|
152
|
-
if (
|
|
153
|
-
this.log('warn', '[SEARCH] No
|
|
150
|
+
if (fileIds.length === 0) {
|
|
151
|
+
this.log('warn', '[SEARCH] No file IDs provided for search, returning empty results');
|
|
154
152
|
return {
|
|
155
153
|
success: true,
|
|
156
154
|
data: { results: [] }
|
|
@@ -158,15 +156,15 @@ export class LangChainRAGService {
|
|
|
158
156
|
}
|
|
159
157
|
|
|
160
158
|
// Load all knowledge vectors in parallel (including temporary documents)
|
|
161
|
-
const knowledgeVectorPromises =
|
|
159
|
+
const knowledgeVectorPromises = fileIds.map(async (fileId) => {
|
|
162
160
|
try {
|
|
163
|
-
const vectorData = await this.loadKnowledgeVectorWithEmbeddings(
|
|
161
|
+
const vectorData = await this.loadKnowledgeVectorWithEmbeddings(fileId);
|
|
164
162
|
return {
|
|
165
|
-
|
|
163
|
+
fileId,
|
|
166
164
|
chunks: vectorData.chunks
|
|
167
165
|
};
|
|
168
166
|
} catch (loadError) {
|
|
169
|
-
this.log('warn', `[SEARCH] Failed to load
|
|
167
|
+
this.log('warn', `[SEARCH] Failed to load file ${fileId}:`, loadError);
|
|
170
168
|
return null;
|
|
171
169
|
}
|
|
172
170
|
});
|
|
@@ -191,10 +189,10 @@ export class LangChainRAGService {
|
|
|
191
189
|
|
|
192
190
|
// Convert results to expected format
|
|
193
191
|
const results: ISearchResult[] = searchResponse.results.map(result => ({
|
|
194
|
-
|
|
192
|
+
fileId: result.metadata?.fileId, // Use fileId directly
|
|
195
193
|
content: result.content,
|
|
196
194
|
score: result.score,
|
|
197
|
-
metadata: result.metadata,
|
|
195
|
+
metadata: result.metadata, // Pass metadata as-is
|
|
198
196
|
chunkId: result.chunkId,
|
|
199
197
|
}));
|
|
200
198
|
|
|
@@ -219,7 +217,7 @@ export class LangChainRAGService {
|
|
|
219
217
|
* Remove knowledge from index
|
|
220
218
|
*/
|
|
221
219
|
async removeKnowledge(fileId: string | number): Promise<boolean> {
|
|
222
|
-
this.log('log', `[REMOVE] Starting removal of
|
|
220
|
+
this.log('log', `[REMOVE] Starting removal of file: ${fileId}`);
|
|
223
221
|
|
|
224
222
|
try {
|
|
225
223
|
await this.ensureInitialized();
|
|
@@ -228,15 +226,15 @@ export class LangChainRAGService {
|
|
|
228
226
|
const vectorFilePath = this.getKnowledgeVectorPath(fileId);
|
|
229
227
|
if (fs.existsSync(vectorFilePath)) {
|
|
230
228
|
fs.unlinkSync(vectorFilePath);
|
|
231
|
-
this.log('log', `[REMOVE] Successfully removed vector file for
|
|
229
|
+
this.log('log', `[REMOVE] Successfully removed vector file for file ${fileId}`);
|
|
232
230
|
return true;
|
|
233
231
|
} else {
|
|
234
|
-
this.log('warn', `[REMOVE] Vector file not found for
|
|
232
|
+
this.log('warn', `[REMOVE] Vector file not found for file ${fileId}`);
|
|
235
233
|
return true; // Consider it successful if file doesn't exist
|
|
236
234
|
}
|
|
237
235
|
|
|
238
236
|
} catch (error: any) {
|
|
239
|
-
this.log('error', `[REMOVE] Failed to remove
|
|
237
|
+
this.log('error', `[REMOVE] Failed to remove file ${fileId}:`, error);
|
|
240
238
|
return false;
|
|
241
239
|
}
|
|
242
240
|
}
|
|
@@ -320,8 +318,8 @@ export class LangChainRAGService {
|
|
|
320
318
|
/**
|
|
321
319
|
* Save chunks to knowledge-specific vector file with embeddings
|
|
322
320
|
*/
|
|
323
|
-
private async saveKnowledgeVector(
|
|
324
|
-
const vectorFilePath = this.getKnowledgeVectorPath(
|
|
321
|
+
private async saveKnowledgeVector(fileId: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }>): Promise<void> {
|
|
322
|
+
const vectorFilePath = this.getKnowledgeVectorPath(fileId);
|
|
325
323
|
const vectorDir = path.dirname(vectorFilePath);
|
|
326
324
|
|
|
327
325
|
// Ensure directory exists
|
|
@@ -331,16 +329,16 @@ export class LangChainRAGService {
|
|
|
331
329
|
|
|
332
330
|
try {
|
|
333
331
|
const vectorData = {
|
|
334
|
-
|
|
332
|
+
fileId,
|
|
335
333
|
chunks,
|
|
336
334
|
timestamp: new Date().toISOString()
|
|
337
335
|
};
|
|
338
336
|
|
|
339
337
|
fs.writeFileSync(vectorFilePath, JSON.stringify(vectorData, null, 2));
|
|
340
|
-
this.log('debug', `[SAVE] Successfully saved ${chunks.length} chunks with embeddings for
|
|
338
|
+
this.log('debug', `[SAVE] Successfully saved ${chunks.length} chunks with embeddings for file ${fileId}`);
|
|
341
339
|
|
|
342
340
|
} catch (error) {
|
|
343
|
-
this.log('error', `[SAVE] Failed to save vector data for
|
|
341
|
+
this.log('error', `[SAVE] Failed to save vector data for file ${fileId}:`, error);
|
|
344
342
|
throw error;
|
|
345
343
|
}
|
|
346
344
|
}
|
|
@@ -348,24 +346,24 @@ export class LangChainRAGService {
|
|
|
348
346
|
/**
|
|
349
347
|
* Load vector data for a specific knowledge item with embeddings
|
|
350
348
|
*/
|
|
351
|
-
private async loadKnowledgeVectorWithEmbeddings(
|
|
352
|
-
const vectorFilePath = this.getKnowledgeVectorPath(
|
|
349
|
+
private async loadKnowledgeVectorWithEmbeddings(fileId: string | number): Promise<{ fileId?: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }> }> {
|
|
350
|
+
const vectorFilePath = this.getKnowledgeVectorPath(fileId);
|
|
353
351
|
|
|
354
352
|
if (!fs.existsSync(vectorFilePath)) {
|
|
355
|
-
this.log('debug', `[LOAD] No vector file found for
|
|
353
|
+
this.log('debug', `[LOAD] No vector file found for file ${fileId}, skipping...`);
|
|
356
354
|
return { chunks: [] };
|
|
357
355
|
}
|
|
358
356
|
|
|
359
357
|
try {
|
|
360
|
-
this.log('debug', `[LOAD] Loading vector data with embeddings for
|
|
358
|
+
this.log('debug', `[LOAD] Loading vector data with embeddings for file ${fileId} from ${vectorFilePath}`);
|
|
361
359
|
const vectorData = JSON.parse(fs.readFileSync(vectorFilePath, 'utf8'));
|
|
362
360
|
|
|
363
361
|
return {
|
|
364
362
|
chunks: vectorData.chunks || [],
|
|
365
|
-
|
|
363
|
+
fileId
|
|
366
364
|
};
|
|
367
365
|
} catch (error) {
|
|
368
|
-
this.log('error', `[LOAD] Failed to load vector data for
|
|
366
|
+
this.log('error', `[LOAD] Failed to load vector data for file ${fileId}:`, error);
|
|
369
367
|
return { chunks: [] };
|
|
370
368
|
}
|
|
371
369
|
}
|
|
@@ -373,12 +371,12 @@ export class LangChainRAGService {
|
|
|
373
371
|
/**
|
|
374
372
|
* Get the file path for a specific knowledge's vector data
|
|
375
373
|
*/
|
|
376
|
-
private getKnowledgeVectorPath(
|
|
374
|
+
private getKnowledgeVectorPath(fileId: string | number): string {
|
|
377
375
|
const vectorDir = path.join(rwsPath.findRootWorkspacePath(), 'files', 'vectors', 'knowledge');
|
|
378
376
|
if (!fs.existsSync(vectorDir)) {
|
|
379
377
|
fs.mkdirSync(vectorDir, { recursive: true });
|
|
380
378
|
}
|
|
381
|
-
return path.join(vectorDir, `knowledge_${
|
|
379
|
+
return path.join(vectorDir, `knowledge_${fileId}.json`);
|
|
382
380
|
}
|
|
383
381
|
|
|
384
382
|
/**
|
|
@@ -7,6 +7,27 @@ import { BlackLogger } from '@rws-framework/server/nest';
|
|
|
7
7
|
let encoding_for_model: any = null;
|
|
8
8
|
encoding_for_model = tiktoken.encoding_for_model
|
|
9
9
|
|
|
10
|
+
// Singleton tokenizer factory for performance
|
|
11
|
+
class TokenizerFactory {
|
|
12
|
+
private static tokenizers = new Map<string, any>();
|
|
13
|
+
|
|
14
|
+
static getTokenizer(model: string): any {
|
|
15
|
+
if (!this.tokenizers.has(model)) {
|
|
16
|
+
try {
|
|
17
|
+
if (encoding_for_model) {
|
|
18
|
+
this.tokenizers.set(model, encoding_for_model(model));
|
|
19
|
+
} else {
|
|
20
|
+
this.tokenizers.set(model, null);
|
|
21
|
+
}
|
|
22
|
+
} catch (e) {
|
|
23
|
+
console.warn(`Could not load tokenizer for model ${model}`);
|
|
24
|
+
this.tokenizers.set(model, null);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return this.tokenizers.get(model);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
10
31
|
@Injectable()
|
|
11
32
|
export class OpenAIRateLimitingService {
|
|
12
33
|
static readonly DEFAULT_CONFIG: Required<IRateLimitConfig> = {
|
|
@@ -37,16 +58,10 @@ export class OpenAIRateLimitingService {
|
|
|
37
58
|
this.config = { ...this.config, ...config };
|
|
38
59
|
}
|
|
39
60
|
|
|
40
|
-
//
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
this.tokenizer = encoding_for_model(model);
|
|
44
|
-
} else {
|
|
45
|
-
this.tokenizer = null;
|
|
46
|
-
}
|
|
47
|
-
} catch (e) {
|
|
61
|
+
// Use singleton tokenizer factory for performance
|
|
62
|
+
this.tokenizer = TokenizerFactory.getTokenizer(model);
|
|
63
|
+
if (!this.tokenizer) {
|
|
48
64
|
this.logger.warn(`Could not load tokenizer for model ${model}, using character-based estimation`);
|
|
49
|
-
this.tokenizer = null;
|
|
50
65
|
}
|
|
51
66
|
|
|
52
67
|
// Reinitialize queue with new concurrency
|
|
@@ -34,13 +34,18 @@ export class OptimizedVectorSearchService {
|
|
|
34
34
|
const allCandidates: IOptimizedSearchResult[] = [];
|
|
35
35
|
let totalCandidates = 0;
|
|
36
36
|
|
|
37
|
-
// Process all knowledge vectors
|
|
37
|
+
// Process all knowledge vectors with early termination optimization
|
|
38
38
|
const searchPromises = knowledgeVectors.map(async (knowledgeVector) => {
|
|
39
39
|
const candidates: IOptimizedSearchResult[] = [];
|
|
40
40
|
const similarities: number[] = []; // Track all similarities for debugging
|
|
41
|
+
let processedCount = 0;
|
|
41
42
|
|
|
42
|
-
|
|
43
|
+
// Sort chunks by some heuristic to check best candidates first (optional optimization)
|
|
44
|
+
const chunks = knowledgeVector.chunks;
|
|
45
|
+
|
|
46
|
+
for (const chunk of chunks) {
|
|
43
47
|
totalCandidates++;
|
|
48
|
+
processedCount++;
|
|
44
49
|
|
|
45
50
|
if (!chunk.embedding || !Array.isArray(chunk.embedding)) {
|
|
46
51
|
continue;
|
|
@@ -54,35 +59,42 @@ export class OptimizedVectorSearchService {
|
|
|
54
59
|
candidates.push({
|
|
55
60
|
content: chunk.content,
|
|
56
61
|
score: similarity,
|
|
57
|
-
metadata:
|
|
58
|
-
|
|
59
|
-
|
|
62
|
+
metadata: {
|
|
63
|
+
...chunk.metadata,
|
|
64
|
+
fileId: knowledgeVector.fileId // Use fileId directly
|
|
65
|
+
},
|
|
66
|
+
fileId: knowledgeVector.fileId, // Always use the fileId from the knowledgeVector
|
|
67
|
+
chunkId: chunk.metadata?.id || `${knowledgeVector.fileId}_chunk_${Date.now()}`
|
|
60
68
|
});
|
|
61
69
|
}
|
|
62
70
|
}
|
|
63
71
|
|
|
72
|
+
// Sort candidates by score and take top maxResults per source
|
|
73
|
+
const topCandidates = candidates
|
|
74
|
+
.sort((a, b) => b.score - a.score)
|
|
75
|
+
.slice(0, maxResults);
|
|
76
|
+
|
|
64
77
|
// Log similarity statistics for debugging
|
|
65
78
|
if (similarities.length > 0) {
|
|
66
79
|
const maxSim = Math.max(...similarities);
|
|
67
80
|
const avgSim = similarities.reduce((a, b) => a + b, 0) / similarities.length;
|
|
68
|
-
console.log(`[VECTOR SEARCH]
|
|
81
|
+
console.log(`[VECTOR SEARCH] File ${knowledgeVector.fileId}: Max similarity: ${maxSim.toFixed(4)}, Avg: ${avgSim.toFixed(4)}, Candidates above ${threshold}: ${candidates.length}, Top results taken: ${topCandidates.length}`);
|
|
69
82
|
}
|
|
70
83
|
|
|
71
|
-
return
|
|
84
|
+
return topCandidates;
|
|
72
85
|
});
|
|
73
86
|
|
|
74
87
|
// Wait for all searches to complete
|
|
75
88
|
const allCandidateArrays = await Promise.all(searchPromises);
|
|
76
89
|
|
|
77
|
-
// Flatten results
|
|
90
|
+
// Flatten results (each source already limited to maxResults)
|
|
78
91
|
for (const candidates of allCandidateArrays) {
|
|
79
92
|
allCandidates.push(...candidates);
|
|
80
93
|
}
|
|
81
94
|
|
|
82
|
-
// Sort by similarity score
|
|
95
|
+
// Sort by similarity score (no additional limiting since each source is already limited)
|
|
83
96
|
const results = allCandidates
|
|
84
|
-
.sort((a, b) => b.score - a.score)
|
|
85
|
-
.slice(0, maxResults);
|
|
97
|
+
.sort((a, b) => b.score - a.score);
|
|
86
98
|
|
|
87
99
|
const searchTime = Date.now() - startTime;
|
|
88
100
|
|
|
@@ -122,7 +134,7 @@ export class OptimizedVectorSearchService {
|
|
|
122
134
|
async batchSearch(
|
|
123
135
|
queries: string[],
|
|
124
136
|
knowledgeVectors: Array<{
|
|
125
|
-
|
|
137
|
+
fileId: string | number;
|
|
126
138
|
chunks: Array<{
|
|
127
139
|
content: string;
|
|
128
140
|
embedding: number[];
|
|
@@ -165,7 +177,7 @@ export class OptimizedVectorSearchService {
|
|
|
165
177
|
private async searchWithEmbedding(request: {
|
|
166
178
|
queryEmbedding: number[];
|
|
167
179
|
knowledgeVectors: Array<{
|
|
168
|
-
|
|
180
|
+
fileId: string | number;
|
|
169
181
|
chunks: Array<{
|
|
170
182
|
content: string;
|
|
171
183
|
embedding: number[];
|
|
@@ -200,8 +212,8 @@ export class OptimizedVectorSearchService {
|
|
|
200
212
|
content: chunk.content,
|
|
201
213
|
score: similarity,
|
|
202
214
|
metadata: chunk.metadata,
|
|
203
|
-
|
|
204
|
-
chunkId: chunk.metadata?.id || `${knowledgeVector.
|
|
215
|
+
fileId: knowledgeVector.fileId,
|
|
216
|
+
chunkId: chunk.metadata?.id || `${knowledgeVector.fileId}_chunk_${Date.now()}`
|
|
205
217
|
});
|
|
206
218
|
}
|
|
207
219
|
}
|
|
@@ -252,7 +264,7 @@ export class OptimizedVectorSearchService {
|
|
|
252
264
|
* Search similar documents (compatibility method from LangChainVectorStoreService)
|
|
253
265
|
*/
|
|
254
266
|
async searchSimilarCompat(request: IVectorSearchRequest, knowledgeVectors: Array<{
|
|
255
|
-
|
|
267
|
+
fileId: string | number;
|
|
256
268
|
chunks: Array<{
|
|
257
269
|
content: string;
|
|
258
270
|
embedding: number[];
|
|
@@ -271,9 +283,9 @@ export class OptimizedVectorSearchService {
|
|
|
271
283
|
let filteredVectors = knowledgeVectors;
|
|
272
284
|
if (filter) {
|
|
273
285
|
filteredVectors = knowledgeVectors.filter(vector => {
|
|
274
|
-
// Check
|
|
275
|
-
if (filter.
|
|
276
|
-
return filter.
|
|
286
|
+
// Check file IDs
|
|
287
|
+
if (filter.fileIds && filter.fileIds.length > 0) {
|
|
288
|
+
return filter.fileIds.includes(String(vector.fileId));
|
|
277
289
|
}
|
|
278
290
|
return true;
|
|
279
291
|
});
|
|
@@ -293,7 +305,7 @@ export class OptimizedVectorSearchService {
|
|
|
293
305
|
score: result.score,
|
|
294
306
|
metadata: result.metadata,
|
|
295
307
|
chunkId: result.chunkId,
|
|
296
|
-
|
|
308
|
+
fileId: result.fileId
|
|
297
309
|
}));
|
|
298
310
|
|
|
299
311
|
return {
|
|
@@ -313,7 +325,7 @@ export class OptimizedVectorSearchService {
|
|
|
313
325
|
* Get search statistics
|
|
314
326
|
*/
|
|
315
327
|
getStats(knowledgeVectors: Array<{
|
|
316
|
-
|
|
328
|
+
fileId: string | number;
|
|
317
329
|
chunks: Array<{ content: string; embedding: number[]; metadata: any; }>;
|
|
318
330
|
}>): { totalChunks: number; totalKnowledge: number } {
|
|
319
331
|
const totalChunks = knowledgeVectors.reduce((total, vector) => total + vector.chunks.length, 0);
|
|
@@ -197,8 +197,9 @@ export class TextChunker {
|
|
|
197
197
|
for (let i = 0; i < chunks.length; i++) {
|
|
198
198
|
const chunk = chunks[i];
|
|
199
199
|
|
|
200
|
-
//
|
|
201
|
-
const
|
|
200
|
+
// Use array for efficient string building
|
|
201
|
+
const parts = currentChunk ? [currentChunk, chunk] : [chunk];
|
|
202
|
+
const combined = parts.join(' ');
|
|
202
203
|
|
|
203
204
|
if (combined.length <= maxChars) {
|
|
204
205
|
// Can merge
|
package/src/types/IPrompt.ts
CHANGED
package/src/types/rag.types.ts
CHANGED
|
@@ -26,7 +26,7 @@ export interface IRAGSearchRequest {
|
|
|
26
26
|
threshold?: number;
|
|
27
27
|
temporaryDocumentSearch?: boolean; // Flag for searching temporary documents (web search)
|
|
28
28
|
filter?: {
|
|
29
|
-
|
|
29
|
+
fileIds?: (string | number)[];
|
|
30
30
|
documentIds?: (string | number)[];
|
|
31
31
|
[key: string]: any;
|
|
32
32
|
};
|
|
@@ -5,7 +5,7 @@ export interface ISearchResult {
|
|
|
5
5
|
content: string;
|
|
6
6
|
score: number;
|
|
7
7
|
metadata: any;
|
|
8
|
-
|
|
8
|
+
fileId: string | number;
|
|
9
9
|
chunkId: string;
|
|
10
10
|
}
|
|
11
11
|
|
|
@@ -30,7 +30,7 @@ export interface IVectorSearchResponse {
|
|
|
30
30
|
export interface IOptimizedSearchRequest {
|
|
31
31
|
query: string;
|
|
32
32
|
knowledgeVectors: Array<{
|
|
33
|
-
|
|
33
|
+
fileId: string | number;
|
|
34
34
|
chunks: Array<{
|
|
35
35
|
content: string;
|
|
36
36
|
embedding: number[];
|
|
@@ -45,7 +45,7 @@ export interface IOptimizedSearchResult {
|
|
|
45
45
|
content: string;
|
|
46
46
|
score: number;
|
|
47
47
|
metadata: any;
|
|
48
|
-
|
|
48
|
+
fileId: string | number;
|
|
49
49
|
chunkId: string;
|
|
50
50
|
}
|
|
51
51
|
|