@futdevpro/nts-dynamo 1.11.1 → 1.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/_modules/discord-assistant/_collections/dias.util.d.ts +6 -0
- package/build/_modules/discord-assistant/_collections/dias.util.d.ts.map +1 -0
- package/build/_modules/discord-assistant/_collections/dias.util.js +36 -0
- package/build/_modules/discord-assistant/_collections/dias.util.js.map +1 -0
- package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.d.ts +1 -2
- package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.d.ts.map +1 -1
- package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.js +16 -10
- package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.js.map +1 -1
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts +9 -5
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts.map +1 -1
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js +18 -6
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js.map +1 -1
- package/build/_modules/discord-assistant/_services/dias-io.control-service.d.ts.map +1 -1
- package/build/_modules/discord-assistant/_services/dias-io.control-service.js +2 -1
- package/build/_modules/discord-assistant/_services/dias-io.control-service.js.map +1 -1
- package/build/_modules/discord-assistant/_services/dias.service-base.d.ts +0 -2
- package/build/_modules/discord-assistant/_services/dias.service-base.d.ts.map +1 -1
- package/build/_modules/discord-assistant/_services/dias.service-base.js +0 -28
- package/build/_modules/discord-assistant/_services/dias.service-base.js.map +1 -1
- package/build/_modules/open-ai/_collections/oai-chunking.util.d.ts +3 -3
- package/build/_modules/open-ai/_collections/oai-chunking.util.d.ts.map +1 -1
- package/build/_modules/open-ai/_collections/oai-chunking.util.js +36 -24
- package/build/_modules/open-ai/_collections/oai-chunking.util.js.map +1 -1
- package/build/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.d.ts +3 -3
- package/build/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.d.ts.map +1 -1
- package/build/_modules/open-ai/_models/interfaces/oai-document-page.interface.d.ts +9 -10
- package/build/_modules/open-ai/_models/interfaces/oai-document-page.interface.d.ts.map +1 -1
- package/build/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.d.ts +7 -9
- package/build/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.d.ts.map +1 -1
- package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.d.ts +25 -15
- package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.d.ts.map +1 -1
- package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.js +73 -65
- package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.js.map +1 -1
- package/build/_modules/open-ai/_services/oai-chunk.service-base.d.ts +10 -9
- package/build/_modules/open-ai/_services/oai-chunk.service-base.d.ts.map +1 -1
- package/build/_modules/open-ai/_services/oai-chunk.service-base.js +18 -33
- package/build/_modules/open-ai/_services/oai-chunk.service-base.js.map +1 -1
- package/build/_modules/open-ai/_services/oai-llm-chat.service-base.d.ts +20 -0
- package/build/_modules/open-ai/_services/oai-llm-chat.service-base.d.ts.map +1 -1
- package/build/_modules/open-ai/_services/oai-llm-chat.service-base.js +12 -0
- package/build/_modules/open-ai/_services/oai-llm-chat.service-base.js.map +1 -1
- package/build/_modules/open-ai/_services/oai-llm.service-base.d.ts +4 -1
- package/build/_modules/open-ai/_services/oai-llm.service-base.d.ts.map +1 -1
- package/build/_modules/open-ai/_services/oai-llm.service-base.js +3 -0
- package/build/_modules/open-ai/_services/oai-llm.service-base.js.map +1 -1
- package/build/_modules/open-ai/_services/oai-vector-data.service.d.ts +4 -1
- package/build/_modules/open-ai/_services/oai-vector-data.service.d.ts.map +1 -1
- package/build/_modules/open-ai/_services/oai-vector-data.service.js +80 -24
- package/build/_modules/open-ai/_services/oai-vector-data.service.js.map +1 -1
- package/build/_modules/open-ai/index.d.ts +2 -0
- package/build/_modules/open-ai/index.d.ts.map +1 -1
- package/build/_modules/open-ai/index.js +2 -0
- package/build/_modules/open-ai/index.js.map +1 -1
- package/build/_services/base/data.service.d.ts +6 -0
- package/build/_services/base/data.service.d.ts.map +1 -1
- package/build/_services/base/data.service.js +6 -0
- package/build/_services/base/data.service.js.map +1 -1
- package/build/_services/base/db.service.d.ts +10 -0
- package/build/_services/base/db.service.d.ts.map +1 -1
- package/build/_services/base/db.service.js +10 -0
- package/build/_services/base/db.service.js.map +1 -1
- package/package.json +3 -3
- package/src/_modules/discord-assistant/_collections/dias.util.ts +43 -0
- package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +12 -5
- package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +33 -16
- package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +6 -1
- package/src/_modules/discord-assistant/_services/dias.service-base.ts +0 -32
- package/src/_modules/open-ai/_collections/oai-chunking.util.ts +51 -35
- package/src/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.ts +6 -3
- package/src/_modules/open-ai/_models/interfaces/oai-document-page.interface.ts +13 -10
- package/src/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.ts +10 -13
- package/src/_modules/open-ai/_models/oai-doc-chunk.data-model.ts +81 -72
- package/src/_modules/open-ai/_services/oai-chunk.service-base.ts +36 -39
- package/src/_modules/open-ai/_services/oai-llm-chat.service-base.ts +12 -0
- package/src/_modules/open-ai/_services/oai-llm.service-base.ts +4 -1
- package/src/_modules/open-ai/_services/oai-vector-data.service.ts +124 -37
- package/src/_modules/open-ai/index.ts +2 -0
- package/src/_services/base/data.service.ts +6 -0
- package/src/_services/base/db.service.ts +10 -0
|
@@ -1,36 +1,34 @@
|
|
|
1
|
-
import { DyFM_Array, DyFM_DataModel_Params, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
|
|
2
|
-
import { DyNTS_OAI_DocChunk, DyNTS_OAI_docChunk_dataParams } from '../_models/oai-doc-chunk.data-model';
|
|
3
|
-
import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
|
|
1
|
+
import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
|
|
4
2
|
import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
|
|
5
|
-
import {
|
|
6
|
-
import { DyNTS_OAI_CompareResult_Type } from '../_enums/oai-compare-result-type.enum';
|
|
7
|
-
import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
|
|
8
|
-
import { DyNTS_OAI_ChunkCompareResult } from '../_models/interfaces/oai-chunk-compare-result.interface';
|
|
3
|
+
import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
|
|
9
4
|
import { DyNTS_OAI_Chunking_Util } from '../_collections/oai-chunking.util';
|
|
10
|
-
import {
|
|
5
|
+
import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
|
|
6
|
+
import { DyNTS_OAI_ChunkCompareResult, DyNTS_OAI_CompareResult_Type, DyNTS_OAI_DocumentPage } from '../../discord-assistant';
|
|
7
|
+
import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
|
|
11
8
|
|
|
12
9
|
|
|
13
|
-
export class
|
|
10
|
+
export class DyNTS_OAI_Chunk_DataServiceBase<
|
|
11
|
+
T_Page extends DyNTS_OAI_DocumentPage<T_Chunk>,
|
|
12
|
+
T_Chunk extends DyNTS_OAI_Chunk
|
|
13
|
+
> extends DyNTS_OAI_VectorDataService<T_Chunk> {
|
|
14
14
|
|
|
15
|
-
constructor(
|
|
15
|
+
constructor(
|
|
16
|
+
dataParams: DyFM_DataModel_Params<T_Chunk>,
|
|
17
|
+
openAiSettings: DyFM_OpenAI_Settings,
|
|
18
|
+
issuer: string
|
|
19
|
+
) {
|
|
16
20
|
super(
|
|
17
21
|
null,
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
openAIConfig: {
|
|
21
|
-
apiKey: DyNTS_global_settings.env_settings.openAi.apiKey,
|
|
22
|
-
organization: DyNTS_global_settings.env_settings.openAi.organization,
|
|
23
|
-
project: DyNTS_global_settings.env_settings.openAi.project,
|
|
24
|
-
},
|
|
25
|
-
defaultSettings: {
|
|
26
|
-
useModel: DyFM_OpenAIModel.textEmbedding_3Large,
|
|
27
|
-
},
|
|
28
|
-
}),
|
|
22
|
+
dataParams,
|
|
23
|
+
openAiSettings,
|
|
29
24
|
issuer
|
|
30
25
|
);
|
|
31
26
|
}
|
|
32
27
|
|
|
33
|
-
async updatePage(
|
|
28
|
+
async updatePage(
|
|
29
|
+
pageCompareResult: DyNTS_OAI_PageCompareResult<T_Page, T_Chunk>,
|
|
30
|
+
issuer: string
|
|
31
|
+
): Promise<void> {
|
|
34
32
|
try {
|
|
35
33
|
// save chunks
|
|
36
34
|
// (only new and modified pages
|
|
@@ -71,14 +69,13 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
|
|
|
71
69
|
}
|
|
72
70
|
}
|
|
73
71
|
|
|
74
|
-
async comparePage(page:
|
|
72
|
+
async comparePage(page: T_Page, issuer: string): Promise<DyNTS_OAI_PageCompareResult<T_Page, T_Chunk>> {
|
|
75
73
|
try {
|
|
76
74
|
const oldChunks = await this.findDataList({
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
});
|
|
75
|
+
allFlaggedParentsMerged: page.allFlaggedParentsMerged,
|
|
76
|
+
} as DyFM_DBFilterSimple<T_Chunk>);
|
|
80
77
|
|
|
81
|
-
const compareResults: DyNTS_OAI_ChunkCompareResult[] = await this.compareChunks(
|
|
78
|
+
const compareResults: DyNTS_OAI_ChunkCompareResult<T_Chunk>[] = await this.compareChunks(
|
|
82
79
|
page.chunks,
|
|
83
80
|
oldChunks
|
|
84
81
|
);
|
|
@@ -138,18 +135,17 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
|
|
|
138
135
|
}
|
|
139
136
|
|
|
140
137
|
async compareChunks(
|
|
141
|
-
chunks:
|
|
142
|
-
oldChunks?:
|
|
143
|
-
): Promise<DyNTS_OAI_ChunkCompareResult[]> {
|
|
138
|
+
chunks: T_Chunk[],
|
|
139
|
+
oldChunks?: T_Chunk[]
|
|
140
|
+
): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>[]> {
|
|
144
141
|
try {
|
|
145
142
|
if (chunks.length === 0) {
|
|
146
143
|
return [];
|
|
147
144
|
}
|
|
148
145
|
|
|
149
146
|
oldChunks ??= await this.findDataList({
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
});
|
|
147
|
+
allFlaggedParentsMerged: chunks[0].allFlaggedParentsMerged,
|
|
148
|
+
} as DyFM_DBFilterSimple<T_Chunk>);
|
|
153
149
|
oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
|
154
150
|
|
|
155
151
|
return await DyFM_Array.asyncForEachAllAtOnce(
|
|
@@ -167,15 +163,14 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
|
|
|
167
163
|
}
|
|
168
164
|
|
|
169
165
|
async compareChunk(
|
|
170
|
-
newChunk:
|
|
171
|
-
oldChunk?:
|
|
172
|
-
): Promise<DyNTS_OAI_ChunkCompareResult
|
|
166
|
+
newChunk: T_Chunk,
|
|
167
|
+
oldChunk?: T_Chunk
|
|
168
|
+
): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>> {
|
|
173
169
|
try {
|
|
174
170
|
oldChunk ??= await this.findData({
|
|
175
|
-
|
|
176
|
-
documentPageId: newChunk.documentPageId,
|
|
171
|
+
allFlaggedParentsMerged: newChunk.allFlaggedParentsMerged,
|
|
177
172
|
chunkIndex: newChunk.chunkIndex,
|
|
178
|
-
});
|
|
173
|
+
} as DyFM_DBFilterSimple<T_Chunk>);
|
|
179
174
|
|
|
180
175
|
/* DyFM_Log.H_info(`✅ Comparing chunk: ${newChunk.chunkIndex}`, {
|
|
181
176
|
newChunk: newChunk,
|
|
@@ -209,6 +204,8 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
|
|
|
209
204
|
errorCode: 'DCH-CSB-CC1',
|
|
210
205
|
});
|
|
211
206
|
}
|
|
207
|
+
|
|
208
|
+
|
|
212
209
|
}
|
|
213
210
|
|
|
214
211
|
}
|
|
@@ -55,6 +55,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
55
55
|
|
|
56
56
|
return answer.toUpperCase().includes(this.predefinedRequests.yesNo.upperCaseYes);
|
|
57
57
|
}
|
|
58
|
+
askYesNoQuestionInConversation = this.yesNoQuestionInConversation;
|
|
58
59
|
|
|
59
60
|
/**
|
|
60
61
|
* Asks the AI to answer a simple question
|
|
@@ -81,6 +82,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
81
82
|
replaceThisInLog
|
|
82
83
|
);
|
|
83
84
|
}
|
|
85
|
+
askSimpleQuestionInConversation = this.simpleQuestionInConversation;
|
|
84
86
|
|
|
85
87
|
/**
|
|
86
88
|
* Asks the AI to answer a percentage question
|
|
@@ -120,6 +122,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
120
122
|
|
|
121
123
|
return +answer;
|
|
122
124
|
}
|
|
125
|
+
askPercentageQuestionInConversation = this.percentageQuestionInConversation;
|
|
123
126
|
|
|
124
127
|
|
|
125
128
|
/**
|
|
@@ -163,6 +166,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
163
166
|
|
|
164
167
|
return null;
|
|
165
168
|
}
|
|
169
|
+
askSelectQuestionInConversation = this.selectQuestionInConversation;
|
|
166
170
|
|
|
167
171
|
/**
|
|
168
172
|
* Asks the AI to select one of the options from the list
|
|
@@ -200,6 +204,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
200
204
|
|
|
201
205
|
return DyFM_safeParseJSON<T>(answer);
|
|
202
206
|
}
|
|
207
|
+
askRequestSelectInConversation = this.requestSelectInConversation;
|
|
203
208
|
|
|
204
209
|
/**
|
|
205
210
|
* Asks the AI to select one or more of the options from the list
|
|
@@ -243,6 +248,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
243
248
|
|
|
244
249
|
return result;
|
|
245
250
|
}
|
|
251
|
+
askMultipleSelectQuestionWithOptionsInConversation = this.multipleSelectQuestionWithOptionsInConversation;
|
|
246
252
|
|
|
247
253
|
/**
|
|
248
254
|
* Asks the AI to select one or more of the options from the list
|
|
@@ -280,6 +286,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
280
286
|
|
|
281
287
|
return DyFM_safeParseList<T[]>(answer);
|
|
282
288
|
}
|
|
289
|
+
askRequestMultipleSelectInConversation = this.requestMultipleSelectInConversation;
|
|
283
290
|
|
|
284
291
|
/**
|
|
285
292
|
* Asks the AI to answer a question that must result a JSON object
|
|
@@ -314,6 +321,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
314
321
|
|
|
315
322
|
return DyFM_safeParseJSON<T>(answer);
|
|
316
323
|
}
|
|
324
|
+
askJsonQuestionInConversation = this.jsonQuestionInConversation;
|
|
317
325
|
|
|
318
326
|
/**
|
|
319
327
|
* Asks the AI to answer a question that must result a JSON object with specific key descriptions
|
|
@@ -353,6 +361,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
353
361
|
|
|
354
362
|
return DyFM_safeParseJSON<T>(answer);
|
|
355
363
|
}
|
|
364
|
+
askJsonQuestionWithKeysDescriptionInConversation = this.jsonQuestionWithKeysDescriptionInConversation;
|
|
356
365
|
|
|
357
366
|
/**
|
|
358
367
|
* Asks the AI to answer a question that must result a JSON object with specific keys
|
|
@@ -392,6 +401,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
392
401
|
|
|
393
402
|
return DyFM_safeParseJSON<T>(answer);
|
|
394
403
|
}
|
|
404
|
+
askJsonQuestionWithExactKeysInConversation = this.jsonQuestionWithExactKeysInConversation;
|
|
395
405
|
|
|
396
406
|
/**
|
|
397
407
|
* Asks the AI to answer a question that must result a list of strings
|
|
@@ -426,6 +436,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
426
436
|
|
|
427
437
|
return DyFM_safeParseList(answer);
|
|
428
438
|
}
|
|
439
|
+
askListQuestionInConversation = this.listQuestionInConversation;
|
|
429
440
|
|
|
430
441
|
/**
|
|
431
442
|
* Asks the AI to answer a question
|
|
@@ -491,6 +502,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
|
|
|
491
502
|
}
|
|
492
503
|
getQuestionAnswerInConversation = this.getAnswerInConversation;
|
|
493
504
|
getQuestionInConversation = this.getAnswerInConversation;
|
|
505
|
+
askQuestionInConversation = this.getAnswerInConversation;
|
|
494
506
|
|
|
495
507
|
/**
|
|
496
508
|
* Resolves a conversation
|
|
@@ -47,7 +47,7 @@ export class DyNTS_OAI_LLM_ServiceBase {
|
|
|
47
47
|
return this.defaultSettings.useModel;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
predefinedRequests: DyNTS_OAI_LLM_Predefined_Requests =
|
|
51
51
|
DyNTS_OAI_LLMDefaultPredefined_Requests;
|
|
52
52
|
|
|
53
53
|
constructor(
|
|
@@ -565,6 +565,7 @@ export class DyNTS_OAI_LLM_ServiceBase {
|
|
|
565
565
|
* parses the JSON list, if it fails, it returns the answer as a single item in an array
|
|
566
566
|
*
|
|
567
567
|
* (uses {@link DyFM_safeParseJSON})
|
|
568
|
+
* @deprecated use {@link DyFM_Shared.safeParseList} instead
|
|
568
569
|
*/
|
|
569
570
|
export function DyFM_safeParseList<T extends Array<any>>(
|
|
570
571
|
textedJSON: string,
|
|
@@ -604,6 +605,7 @@ export function DyFM_safeParseList<T extends Array<any>>(
|
|
|
604
605
|
* parses the JSON, if it fails, it returns { unparsableResult: textedJSON }
|
|
605
606
|
*
|
|
606
607
|
* (uses {@link DyFM_failableSafeParseJSON})
|
|
608
|
+
* @deprecated use {@link DyFM_Shared.safeParseJSON} instead
|
|
607
609
|
*/
|
|
608
610
|
export function DyFM_safeParseJSON<T = any>(
|
|
609
611
|
textedJSON: string,
|
|
@@ -629,6 +631,7 @@ export function DyFM_safeParseJSON<T = any>(
|
|
|
629
631
|
/**
|
|
630
632
|
* parses the JSON, if it fails, it throws an error
|
|
631
633
|
* (also will extract the JSON from "```json ... ```" wrapper)
|
|
634
|
+
* @deprecated use {@link DyFM_Shared.failableSafeParseJSON} instead
|
|
632
635
|
*/
|
|
633
636
|
export function DyFM_failableSafeParseJSON<T = any>(textedJSON: string): T {
|
|
634
637
|
if (!textedJSON) {
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
DyFM_Metadata,
|
|
8
8
|
DyFM_sameObjects
|
|
9
9
|
} from '@futdevpro/fsm-dynamo';
|
|
10
|
-
import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
|
|
10
|
+
import { DyFM_OpenAI_Settings, DyFM_OpenAIEmbeddingModel, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
|
|
11
11
|
|
|
12
12
|
import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
|
|
13
13
|
import { DyNTS_DataService } from '../../../_services/base/data.service';
|
|
@@ -58,6 +58,7 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
58
58
|
* (properties that are marked with vectorizeFrom in other properties)
|
|
59
59
|
*/
|
|
60
60
|
readonly vectorizeProperties: DyFM_DataProperty_Params<any>[];
|
|
61
|
+
readonly vectorizedProperties: DyFM_DataProperty_Params<any>[];
|
|
61
62
|
/* readonly vectorizeProperties: { from: string, to: string }[]; */
|
|
62
63
|
|
|
63
64
|
embedding_CS: DyNTS_OAI_Embedding_ControlService;
|
|
@@ -81,11 +82,25 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
81
82
|
issuer: string
|
|
82
83
|
) {
|
|
83
84
|
super(data, dataParams, issuer);
|
|
85
|
+
|
|
86
|
+
this.vectorizedProperties = Object.values(dataParams.properties).filter(
|
|
87
|
+
(property: DyFM_DataProperty_Params<any>) => property.vectorizedFrom
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
this.vectorizeProperties = Object.keys(dataParams.properties).filter(
|
|
91
|
+
(key: string): boolean => this.vectorizedProperties.some(
|
|
92
|
+
(property: DyFM_DataProperty_Params<any>) => property.vectorizedFrom.includes(key)
|
|
93
|
+
)
|
|
94
|
+
).map(
|
|
95
|
+
(key: string): DyFM_DataProperty_Params<any> => dataParams.properties[key] as DyFM_DataProperty_Params<any>
|
|
96
|
+
);
|
|
84
97
|
|
|
85
|
-
const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
|
|
86
|
-
(key: string): boolean => (
|
|
98
|
+
/* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
|
|
99
|
+
(key: string): boolean => (
|
|
100
|
+
dataParams.properties[key] as DyFM_DataProperty_Params<any>
|
|
101
|
+
).vectorizedFrom &&
|
|
87
102
|
(dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
|
|
88
|
-
);
|
|
103
|
+
); */
|
|
89
104
|
|
|
90
105
|
if (!openAISettings.openAIConfig.apiKey) {
|
|
91
106
|
throw new DyFM_Error({
|
|
@@ -96,7 +111,8 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
96
111
|
});
|
|
97
112
|
}
|
|
98
113
|
|
|
99
|
-
const model = openAISettings.defaultSettings.useModel ??
|
|
114
|
+
const model: DyFM_OpenAIEmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OpenAIEmbeddingModel ??
|
|
115
|
+
DyFM_OpenAIModel.textEmbedding_3Large;
|
|
100
116
|
|
|
101
117
|
/* if (
|
|
102
118
|
!DyFM_allOpenAIEmbeddingModels.includes(model)
|
|
@@ -109,7 +125,29 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
109
125
|
});
|
|
110
126
|
} */
|
|
111
127
|
|
|
112
|
-
if (!
|
|
128
|
+
if (!this.vectorizedProperties.length) {
|
|
129
|
+
throw new Error(
|
|
130
|
+
'vectorizedProperties is missing! ' +
|
|
131
|
+
`\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter(
|
|
132
|
+
(key: string): boolean => Boolean(
|
|
133
|
+
(dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
|
|
134
|
+
)
|
|
135
|
+
).map(key => `"${key}"`).join(', ')}`
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (!this.vectorizeProperties.length) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
'vectorizeProperties is missing! ' +
|
|
142
|
+
`\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
|
|
143
|
+
(key: string): boolean => Boolean(
|
|
144
|
+
(dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
|
|
145
|
+
)
|
|
146
|
+
).map(key => `"${key}"`).join(', ')}`
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/* if (!vectorizePropertyKeys.length) {
|
|
113
151
|
throw new Error(
|
|
114
152
|
'vectorizedPropertyKeys is missing! ' +
|
|
115
153
|
`\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
|
|
@@ -118,16 +156,20 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
118
156
|
)
|
|
119
157
|
).map(key => `"${key}"`).join(', ')}`
|
|
120
158
|
);
|
|
121
|
-
}
|
|
159
|
+
} */
|
|
122
160
|
|
|
123
|
-
vectorizePropertyKeys.forEach((key: string): void => {
|
|
161
|
+
/* vectorizePropertyKeys.forEach((key: string): void => {
|
|
124
162
|
dataParams.properties[key].embeddingModel ??= model;
|
|
163
|
+
}); */
|
|
164
|
+
|
|
165
|
+
this.vectorizeProperties.forEach((property: DyFM_DataProperty_Params<any>): void => {
|
|
166
|
+
property.embeddingModel ??= model;
|
|
125
167
|
});
|
|
126
168
|
|
|
127
|
-
this.vectorizeProperties = vectorizePropertyKeys.map(
|
|
169
|
+
/* this.vectorizeProperties = vectorizePropertyKeys.map(
|
|
128
170
|
(key: string): DyFM_DataProperty_Params<any> =>
|
|
129
171
|
dataParams.properties[key] as DyFM_DataProperty_Params<any>
|
|
130
|
-
);
|
|
172
|
+
); */
|
|
131
173
|
}
|
|
132
174
|
|
|
133
175
|
/**
|
|
@@ -140,43 +182,22 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
140
182
|
*
|
|
141
183
|
* @returns The vectorized data
|
|
142
184
|
*/
|
|
143
|
-
async vectorizeDataProperties(
|
|
185
|
+
async vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T> {
|
|
144
186
|
try {
|
|
145
187
|
let existingData: T;
|
|
146
188
|
|
|
147
|
-
if (!
|
|
148
|
-
existingData = await this.getDataById(
|
|
189
|
+
if (!forceVectorize) {
|
|
190
|
+
const existingData: T = await this.getDataById(newData._id, true, true);
|
|
149
191
|
}
|
|
150
192
|
|
|
151
193
|
await DyFM_Array.asyncForEachAllAtOnce(
|
|
152
|
-
this.
|
|
194
|
+
this.vectorizedProperties,
|
|
153
195
|
async (property: DyFM_DataProperty_Params<any>) => {
|
|
154
|
-
|
|
155
|
-
data[property.vectorizedFrom] &&
|
|
156
|
-
(!data[property.key]?.length || alwaysVectorize || property.vectorizeAlways ||
|
|
157
|
-
!DyFM_sameObjects(existingData[property.key], data[property.key])
|
|
158
|
-
)
|
|
159
|
-
) {
|
|
160
|
-
if (!property.embeddingModel) {
|
|
161
|
-
throw new DyFM_Error({
|
|
162
|
-
...this.getDefaultErrorSettings(
|
|
163
|
-
'vectorizeDataProperties',
|
|
164
|
-
new Error(`Property "${property.key}" has no embedding model!`)
|
|
165
|
-
),
|
|
166
|
-
|
|
167
|
-
errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
data[property.key] = await this.vectorize(
|
|
172
|
-
data[property.vectorizedFrom],
|
|
173
|
-
property.embeddingModel
|
|
174
|
-
);
|
|
175
|
-
}
|
|
196
|
+
await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
|
|
176
197
|
}
|
|
177
198
|
);
|
|
178
199
|
|
|
179
|
-
return
|
|
200
|
+
return newData;
|
|
180
201
|
} catch (error) {
|
|
181
202
|
throw new DyFM_Error({
|
|
182
203
|
...this.getDefaultErrorSettings('vectorizeDataProperties', error),
|
|
@@ -184,6 +205,72 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
|
|
|
184
205
|
}
|
|
185
206
|
}
|
|
186
207
|
|
|
208
|
+
private async checkAndVectorizeSingleProperty(
|
|
209
|
+
vectorizedProperty: DyFM_DataProperty_Params<any>,
|
|
210
|
+
newData: T,
|
|
211
|
+
existingData: T,
|
|
212
|
+
forceVectorize?: boolean
|
|
213
|
+
): Promise<void> {
|
|
214
|
+
const newValue: string = await this.getVectorizingDataForVectorizedProperty(
|
|
215
|
+
vectorizedProperty,
|
|
216
|
+
newData
|
|
217
|
+
);
|
|
218
|
+
const existingValue: string = await this.getVectorizingDataForVectorizedProperty(
|
|
219
|
+
vectorizedProperty,
|
|
220
|
+
existingData
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
const vectorizedFrom: string = vectorizedProperty.vectorizedFrom[0];
|
|
224
|
+
const vectorizingPropertyHaveValue: boolean = Boolean(newData[vectorizedFrom]);
|
|
225
|
+
const newDataIsNewChanged: boolean =
|
|
226
|
+
(!existingValue && vectorizingPropertyHaveValue) ||
|
|
227
|
+
!DyFM_sameObjects(existingValue, newValue);
|
|
228
|
+
|
|
229
|
+
if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
|
|
230
|
+
if (!vectorizedProperty.embeddingModel) {
|
|
231
|
+
throw new DyFM_Error({
|
|
232
|
+
...this.getDefaultErrorSettings(
|
|
233
|
+
'vectorizeDataProperties',
|
|
234
|
+
new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)
|
|
235
|
+
),
|
|
236
|
+
|
|
237
|
+
errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
newData[vectorizedProperty.key] = await this.vectorize(
|
|
242
|
+
newValue,
|
|
243
|
+
vectorizedProperty.embeddingModel
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
private async getVectorizingDataForVectorizedProperty(
|
|
249
|
+
vectorizedProperty: DyFM_DataProperty_Params<any>,
|
|
250
|
+
data: T,
|
|
251
|
+
/* alwaysVectorize?: boolean */
|
|
252
|
+
): Promise<string> {
|
|
253
|
+
if (vectorizedProperty.vectorizedFrom.length === 1) {
|
|
254
|
+
const value: unknown = data[vectorizedProperty.vectorizedFrom[0]];
|
|
255
|
+
|
|
256
|
+
if (typeof value === 'string') {
|
|
257
|
+
return value;
|
|
258
|
+
} else if (Array.isArray(value) || typeof value === 'object') {
|
|
259
|
+
return JSON.stringify(value);
|
|
260
|
+
} else {
|
|
261
|
+
return `${value}`;
|
|
262
|
+
}
|
|
263
|
+
} else {
|
|
264
|
+
const object: any = {};
|
|
265
|
+
|
|
266
|
+
vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
|
|
267
|
+
object[key] = data[key];
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
return JSON.stringify(object);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
187
274
|
/**
|
|
188
275
|
* Vectorizes the input
|
|
189
276
|
*
|
|
@@ -23,6 +23,7 @@ export * from './_collections/oai-chunking.util';
|
|
|
23
23
|
export * from './_collections/oai-document.util';
|
|
24
24
|
export * from './_collections/oai-global-settings.const';
|
|
25
25
|
export * from './_collections/oai-llm-predefined-requests.conts';
|
|
26
|
+
export * from './_collections/oai-llm-predefined-requests-hu.conts';
|
|
26
27
|
|
|
27
28
|
// ENUMS
|
|
28
29
|
export * from './_enums/oai-compare-result-type.enum';
|
|
@@ -32,6 +33,7 @@ export * from './_enums/oai-gpt-message-role.enum';
|
|
|
32
33
|
export * from './_models/oai-doc-chunk.data-model';
|
|
33
34
|
// models/INTERFACES
|
|
34
35
|
export * from './_models/interfaces/oai-chunk-compare-result.interface';
|
|
36
|
+
export * from './_models/interfaces/oai-page-compare-result.interface';
|
|
35
37
|
export * from './_models/interfaces/oai-document-page.interface';
|
|
36
38
|
export * from './_models/interfaces/oai-global-settings.interface';
|
|
37
39
|
export * from './_models/interfaces/oai-gpt-message.interface';
|
|
@@ -549,6 +549,8 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
|
|
|
549
549
|
* @example
|
|
550
550
|
* // or by id that is in list:
|
|
551
551
|
* { userIds: { $in: this.userId } }
|
|
552
|
+
* // or by userIds:
|
|
553
|
+
* { userId: { $in: userIds } }
|
|
552
554
|
* //
|
|
553
555
|
* @example
|
|
554
556
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -605,6 +607,8 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
|
|
|
605
607
|
* @example
|
|
606
608
|
* // or by id that is in list:
|
|
607
609
|
* { userIds: { $in: this.userId } }
|
|
610
|
+
* // or by userIds:
|
|
611
|
+
* { userId: { $in: userIds } }
|
|
608
612
|
* //
|
|
609
613
|
* @example
|
|
610
614
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -661,6 +665,8 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
|
|
|
661
665
|
* @example
|
|
662
666
|
* // or by id that is in list:
|
|
663
667
|
* { userIds: { $in: this.userId } }
|
|
668
|
+
* // or by userIds:
|
|
669
|
+
* { userId: { $in: userIds } }
|
|
664
670
|
* //
|
|
665
671
|
* @example
|
|
666
672
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -735,6 +735,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
|
|
|
735
735
|
* @example
|
|
736
736
|
* // or by id that is in list:
|
|
737
737
|
* { userIds: { $in: this.userId } }
|
|
738
|
+
* // or by userIds:
|
|
739
|
+
* { userId: { $in: userIds } }
|
|
738
740
|
* //
|
|
739
741
|
* @example
|
|
740
742
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -778,6 +780,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
|
|
|
778
780
|
* @example
|
|
779
781
|
* // or by id that is in list:
|
|
780
782
|
* { userIds: { $in: this.userId } }
|
|
783
|
+
* // or by userIds:
|
|
784
|
+
* { userId: { $in: userIds } }
|
|
781
785
|
* //
|
|
782
786
|
* @example
|
|
783
787
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -825,6 +829,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
|
|
|
825
829
|
* @example
|
|
826
830
|
* // or by id that is in list:
|
|
827
831
|
* { userIds: { $in: this.userId } }
|
|
832
|
+
* // or by userIds:
|
|
833
|
+
* { userId: { $in: userIds } }
|
|
828
834
|
* //
|
|
829
835
|
* @example
|
|
830
836
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -929,6 +935,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
|
|
|
929
935
|
* @example
|
|
930
936
|
* // or by id that is in list:
|
|
931
937
|
* { userIds: { $in: this.userId } }
|
|
938
|
+
* // or by userIds:
|
|
939
|
+
* { userId: { $in: userIds } }
|
|
932
940
|
* //
|
|
933
941
|
* @example
|
|
934
942
|
* // or by number or Date that is Greater Than AND Less Than:
|
|
@@ -1008,6 +1016,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
|
|
|
1008
1016
|
* @example
|
|
1009
1017
|
* // or by id that is in list:
|
|
1010
1018
|
* { userIds: { $in: this.userId } }
|
|
1019
|
+
* // or by userIds:
|
|
1020
|
+
* { userId: { $in: userIds } }
|
|
1011
1021
|
* //
|
|
1012
1022
|
* @example
|
|
1013
1023
|
* // or by number or Date that is Greater Than AND Less Than:
|