@futdevpro/nts-dynamo 1.11.1 → 1.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/build/_modules/discord-assistant/_collections/dias.util.d.ts +6 -0
  2. package/build/_modules/discord-assistant/_collections/dias.util.d.ts.map +1 -0
  3. package/build/_modules/discord-assistant/_collections/dias.util.js +36 -0
  4. package/build/_modules/discord-assistant/_collections/dias.util.js.map +1 -0
  5. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.d.ts +1 -2
  6. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.d.ts.map +1 -1
  7. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.js +16 -10
  8. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.js.map +1 -1
  9. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts +9 -5
  10. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts.map +1 -1
  11. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js +18 -6
  12. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js.map +1 -1
  13. package/build/_modules/discord-assistant/_services/dias-io.control-service.d.ts.map +1 -1
  14. package/build/_modules/discord-assistant/_services/dias-io.control-service.js +2 -1
  15. package/build/_modules/discord-assistant/_services/dias-io.control-service.js.map +1 -1
  16. package/build/_modules/discord-assistant/_services/dias.service-base.d.ts +0 -2
  17. package/build/_modules/discord-assistant/_services/dias.service-base.d.ts.map +1 -1
  18. package/build/_modules/discord-assistant/_services/dias.service-base.js +0 -28
  19. package/build/_modules/discord-assistant/_services/dias.service-base.js.map +1 -1
  20. package/build/_modules/open-ai/_collections/oai-chunking.util.d.ts +3 -3
  21. package/build/_modules/open-ai/_collections/oai-chunking.util.d.ts.map +1 -1
  22. package/build/_modules/open-ai/_collections/oai-chunking.util.js +36 -24
  23. package/build/_modules/open-ai/_collections/oai-chunking.util.js.map +1 -1
  24. package/build/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.d.ts +3 -3
  25. package/build/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.d.ts.map +1 -1
  26. package/build/_modules/open-ai/_models/interfaces/oai-document-page.interface.d.ts +9 -10
  27. package/build/_modules/open-ai/_models/interfaces/oai-document-page.interface.d.ts.map +1 -1
  28. package/build/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.d.ts +7 -9
  29. package/build/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.d.ts.map +1 -1
  30. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.d.ts +25 -15
  31. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.d.ts.map +1 -1
  32. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.js +73 -65
  33. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.js.map +1 -1
  34. package/build/_modules/open-ai/_services/oai-chunk.service-base.d.ts +10 -9
  35. package/build/_modules/open-ai/_services/oai-chunk.service-base.d.ts.map +1 -1
  36. package/build/_modules/open-ai/_services/oai-chunk.service-base.js +18 -33
  37. package/build/_modules/open-ai/_services/oai-chunk.service-base.js.map +1 -1
  38. package/build/_modules/open-ai/_services/oai-llm-chat.service-base.d.ts +20 -0
  39. package/build/_modules/open-ai/_services/oai-llm-chat.service-base.d.ts.map +1 -1
  40. package/build/_modules/open-ai/_services/oai-llm-chat.service-base.js +12 -0
  41. package/build/_modules/open-ai/_services/oai-llm-chat.service-base.js.map +1 -1
  42. package/build/_modules/open-ai/_services/oai-llm.service-base.d.ts +4 -1
  43. package/build/_modules/open-ai/_services/oai-llm.service-base.d.ts.map +1 -1
  44. package/build/_modules/open-ai/_services/oai-llm.service-base.js +3 -0
  45. package/build/_modules/open-ai/_services/oai-llm.service-base.js.map +1 -1
  46. package/build/_modules/open-ai/_services/oai-vector-data.service.d.ts +4 -1
  47. package/build/_modules/open-ai/_services/oai-vector-data.service.d.ts.map +1 -1
  48. package/build/_modules/open-ai/_services/oai-vector-data.service.js +80 -24
  49. package/build/_modules/open-ai/_services/oai-vector-data.service.js.map +1 -1
  50. package/build/_modules/open-ai/index.d.ts +2 -0
  51. package/build/_modules/open-ai/index.d.ts.map +1 -1
  52. package/build/_modules/open-ai/index.js +2 -0
  53. package/build/_modules/open-ai/index.js.map +1 -1
  54. package/build/_services/base/data.service.d.ts +6 -0
  55. package/build/_services/base/data.service.d.ts.map +1 -1
  56. package/build/_services/base/data.service.js +6 -0
  57. package/build/_services/base/data.service.js.map +1 -1
  58. package/build/_services/base/db.service.d.ts +10 -0
  59. package/build/_services/base/db.service.d.ts.map +1 -1
  60. package/build/_services/base/db.service.js +10 -0
  61. package/build/_services/base/db.service.js.map +1 -1
  62. package/package.json +3 -3
  63. package/src/_modules/discord-assistant/_collections/dias.util.ts +43 -0
  64. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +12 -5
  65. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +33 -16
  66. package/src/_modules/discord-assistant/_services/dias-io.control-service.ts +6 -1
  67. package/src/_modules/discord-assistant/_services/dias.service-base.ts +0 -32
  68. package/src/_modules/open-ai/_collections/oai-chunking.util.ts +51 -35
  69. package/src/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.ts +6 -3
  70. package/src/_modules/open-ai/_models/interfaces/oai-document-page.interface.ts +13 -10
  71. package/src/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.ts +10 -13
  72. package/src/_modules/open-ai/_models/oai-doc-chunk.data-model.ts +81 -72
  73. package/src/_modules/open-ai/_services/oai-chunk.service-base.ts +36 -39
  74. package/src/_modules/open-ai/_services/oai-llm-chat.service-base.ts +12 -0
  75. package/src/_modules/open-ai/_services/oai-llm.service-base.ts +4 -1
  76. package/src/_modules/open-ai/_services/oai-vector-data.service.ts +124 -37
  77. package/src/_modules/open-ai/index.ts +2 -0
  78. package/src/_services/base/data.service.ts +6 -0
  79. package/src/_services/base/db.service.ts +10 -0
@@ -1,36 +1,34 @@
1
- import { DyFM_Array, DyFM_DataModel_Params, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
2
- import { DyNTS_OAI_DocChunk, DyNTS_OAI_docChunk_dataParams } from '../_models/oai-doc-chunk.data-model';
3
- import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
1
+ import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
4
2
  import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
5
- import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
6
- import { DyNTS_OAI_CompareResult_Type } from '../_enums/oai-compare-result-type.enum';
7
- import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
8
- import { DyNTS_OAI_ChunkCompareResult } from '../_models/interfaces/oai-chunk-compare-result.interface';
3
+ import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
9
4
  import { DyNTS_OAI_Chunking_Util } from '../_collections/oai-chunking.util';
10
- import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
5
+ import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
6
+ import { DyNTS_OAI_ChunkCompareResult, DyNTS_OAI_CompareResult_Type, DyNTS_OAI_DocumentPage } from '../../discord-assistant';
7
+ import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
11
8
 
12
9
 
13
- export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyNTS_OAI_DocChunk> {
10
+ export class DyNTS_OAI_Chunk_DataServiceBase<
11
+ T_Page extends DyNTS_OAI_DocumentPage<T_Chunk>,
12
+ T_Chunk extends DyNTS_OAI_Chunk
13
+ > extends DyNTS_OAI_VectorDataService<T_Chunk> {
14
14
 
15
- constructor(issuer: string) {
15
+ constructor(
16
+ dataParams: DyFM_DataModel_Params<T_Chunk>,
17
+ openAiSettings: DyFM_OpenAI_Settings,
18
+ issuer: string
19
+ ) {
16
20
  super(
17
21
  null,
18
- DyNTS_OAI_docChunk_dataParams,
19
- new DyFM_OpenAI_Settings({
20
- openAIConfig: {
21
- apiKey: DyNTS_global_settings.env_settings.openAi.apiKey,
22
- organization: DyNTS_global_settings.env_settings.openAi.organization,
23
- project: DyNTS_global_settings.env_settings.openAi.project,
24
- },
25
- defaultSettings: {
26
- useModel: DyFM_OpenAIModel.textEmbedding_3Large,
27
- },
28
- }),
22
+ dataParams,
23
+ openAiSettings,
29
24
  issuer
30
25
  );
31
26
  }
32
27
 
33
- async updatePage(pageCompareResult: DyNTS_OAI_PageCompareResult, issuer: string): Promise<void> {
28
+ async updatePage(
29
+ pageCompareResult: DyNTS_OAI_PageCompareResult<T_Page, T_Chunk>,
30
+ issuer: string
31
+ ): Promise<void> {
34
32
  try {
35
33
  // save chunks
36
34
  // (only new and modified pages
@@ -71,14 +69,13 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
71
69
  }
72
70
  }
73
71
 
74
- async comparePage(page: DyNTS_OAI_DocumentPage, issuer: string): Promise<DyNTS_OAI_PageCompareResult> {
72
+ async comparePage(page: T_Page, issuer: string): Promise<DyNTS_OAI_PageCompareResult<T_Page, T_Chunk>> {
75
73
  try {
76
74
  const oldChunks = await this.findDataList({
77
- documentId: `${page.docId}`,
78
- documentPageId: page._id,
79
- });
75
+ allFlaggedParentsMerged: page.allFlaggedParentsMerged,
76
+ } as DyFM_DBFilterSimple<T_Chunk>);
80
77
 
81
- const compareResults: DyNTS_OAI_ChunkCompareResult[] = await this.compareChunks(
78
+ const compareResults: DyNTS_OAI_ChunkCompareResult<T_Chunk>[] = await this.compareChunks(
82
79
  page.chunks,
83
80
  oldChunks
84
81
  );
@@ -138,18 +135,17 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
138
135
  }
139
136
 
140
137
  async compareChunks(
141
- chunks: DyNTS_OAI_DocChunk[],
142
- oldChunks?: DyNTS_OAI_DocChunk[]
143
- ): Promise<DyNTS_OAI_ChunkCompareResult[]> {
138
+ chunks: T_Chunk[],
139
+ oldChunks?: T_Chunk[]
140
+ ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>[]> {
144
141
  try {
145
142
  if (chunks.length === 0) {
146
143
  return [];
147
144
  }
148
145
 
149
146
  oldChunks ??= await this.findDataList({
150
- documentId: chunks[0].documentId,
151
- documentPageId: chunks[0].documentPageId,
152
- });
147
+ allFlaggedParentsMerged: chunks[0].allFlaggedParentsMerged,
148
+ } as DyFM_DBFilterSimple<T_Chunk>);
153
149
  oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
154
150
 
155
151
  return await DyFM_Array.asyncForEachAllAtOnce(
@@ -167,15 +163,14 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
167
163
  }
168
164
 
169
165
  async compareChunk(
170
- newChunk: DyNTS_OAI_DocChunk,
171
- oldChunk?: DyNTS_OAI_DocChunk
172
- ): Promise<DyNTS_OAI_ChunkCompareResult> {
166
+ newChunk: T_Chunk,
167
+ oldChunk?: T_Chunk
168
+ ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>> {
173
169
  try {
174
170
  oldChunk ??= await this.findData({
175
- documentId: newChunk.documentId,
176
- documentPageId: newChunk.documentPageId,
171
+ allFlaggedParentsMerged: newChunk.allFlaggedParentsMerged,
177
172
  chunkIndex: newChunk.chunkIndex,
178
- });
173
+ } as DyFM_DBFilterSimple<T_Chunk>);
179
174
 
180
175
  /* DyFM_Log.H_info(`✅ Comparing chunk: ${newChunk.chunkIndex}`, {
181
176
  newChunk: newChunk,
@@ -209,6 +204,8 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
209
204
  errorCode: 'DCH-CSB-CC1',
210
205
  });
211
206
  }
207
+
208
+
212
209
  }
213
210
 
214
211
  }
@@ -55,6 +55,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
55
55
 
56
56
  return answer.toUpperCase().includes(this.predefinedRequests.yesNo.upperCaseYes);
57
57
  }
58
+ askYesNoQuestionInConversation = this.yesNoQuestionInConversation;
58
59
 
59
60
  /**
60
61
  * Asks the AI to answer a simple question
@@ -81,6 +82,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
81
82
  replaceThisInLog
82
83
  );
83
84
  }
85
+ askSimpleQuestionInConversation = this.simpleQuestionInConversation;
84
86
 
85
87
  /**
86
88
  * Asks the AI to answer a percentage question
@@ -120,6 +122,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
120
122
 
121
123
  return +answer;
122
124
  }
125
+ askPercentageQuestionInConversation = this.percentageQuestionInConversation;
123
126
 
124
127
 
125
128
  /**
@@ -163,6 +166,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
163
166
 
164
167
  return null;
165
168
  }
169
+ askSelectQuestionInConversation = this.selectQuestionInConversation;
166
170
 
167
171
  /**
168
172
  * Asks the AI to select one of the options from the list
@@ -200,6 +204,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
200
204
 
201
205
  return DyFM_safeParseJSON<T>(answer);
202
206
  }
207
+ askRequestSelectInConversation = this.requestSelectInConversation;
203
208
 
204
209
  /**
205
210
  * Asks the AI to select one or more of the options from the list
@@ -243,6 +248,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
243
248
 
244
249
  return result;
245
250
  }
251
+ askMultipleSelectQuestionWithOptionsInConversation = this.multipleSelectQuestionWithOptionsInConversation;
246
252
 
247
253
  /**
248
254
  * Asks the AI to select one or more of the options from the list
@@ -280,6 +286,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
280
286
 
281
287
  return DyFM_safeParseList<T[]>(answer);
282
288
  }
289
+ askRequestMultipleSelectInConversation = this.requestMultipleSelectInConversation;
283
290
 
284
291
  /**
285
292
  * Asks the AI to answer a question that must result a JSON object
@@ -314,6 +321,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
314
321
 
315
322
  return DyFM_safeParseJSON<T>(answer);
316
323
  }
324
+ askJsonQuestionInConversation = this.jsonQuestionInConversation;
317
325
 
318
326
  /**
319
327
  * Asks the AI to answer a question that must result a JSON object with specific key descriptions
@@ -353,6 +361,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
353
361
 
354
362
  return DyFM_safeParseJSON<T>(answer);
355
363
  }
364
+ askJsonQuestionWithKeysDescriptionInConversation = this.jsonQuestionWithKeysDescriptionInConversation;
356
365
 
357
366
  /**
358
367
  * Asks the AI to answer a question that must result a JSON object with specific keys
@@ -392,6 +401,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
392
401
 
393
402
  return DyFM_safeParseJSON<T>(answer);
394
403
  }
404
+ askJsonQuestionWithExactKeysInConversation = this.jsonQuestionWithExactKeysInConversation;
395
405
 
396
406
  /**
397
407
  * Asks the AI to answer a question that must result a list of strings
@@ -426,6 +436,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
426
436
 
427
437
  return DyFM_safeParseList(answer);
428
438
  }
439
+ askListQuestionInConversation = this.listQuestionInConversation;
429
440
 
430
441
  /**
431
442
  * Asks the AI to answer a question
@@ -491,6 +502,7 @@ export class DyNTS_OAI_LLMChat_ServiceBase extends DyNTS_OAI_LLM_ServiceBase {
491
502
  }
492
503
  getQuestionAnswerInConversation = this.getAnswerInConversation;
493
504
  getQuestionInConversation = this.getAnswerInConversation;
505
+ askQuestionInConversation = this.getAnswerInConversation;
494
506
 
495
507
  /**
496
508
  * Resolves a conversation
@@ -47,7 +47,7 @@ export class DyNTS_OAI_LLM_ServiceBase {
47
47
  return this.defaultSettings.useModel;
48
48
  }
49
49
 
50
- readonly predefinedRequests: DyNTS_OAI_LLM_Predefined_Requests =
50
+ predefinedRequests: DyNTS_OAI_LLM_Predefined_Requests =
51
51
  DyNTS_OAI_LLMDefaultPredefined_Requests;
52
52
 
53
53
  constructor(
@@ -565,6 +565,7 @@ export class DyNTS_OAI_LLM_ServiceBase {
565
565
  * parses the JSON list, if it fails, it returns the answer as a single item in an array
566
566
  *
567
567
  * (uses {@link DyFM_safeParseJSON})
568
+ * @deprecated use {@link DyFM_Shared.safeParseList} instead
568
569
  */
569
570
  export function DyFM_safeParseList<T extends Array<any>>(
570
571
  textedJSON: string,
@@ -604,6 +605,7 @@ export function DyFM_safeParseList<T extends Array<any>>(
604
605
  * parses the JSON, if it fails, it returns { unparsableResult: textedJSON }
605
606
  *
606
607
  * (uses {@link DyFM_failableSafeParseJSON})
608
+ * @deprecated use {@link DyFM_Shared.safeParseJSON} instead
607
609
  */
608
610
  export function DyFM_safeParseJSON<T = any>(
609
611
  textedJSON: string,
@@ -629,6 +631,7 @@ export function DyFM_safeParseJSON<T = any>(
629
631
  /**
630
632
  * parses the JSON, if it fails, it throws an error
631
633
  * (also will extract the JSON from "```json ... ```" wrapper)
634
+ * @deprecated use {@link DyFM_Shared.failableSafeParseJSON} instead
632
635
  */
633
636
  export function DyFM_failableSafeParseJSON<T = any>(textedJSON: string): T {
634
637
  if (!textedJSON) {
@@ -7,7 +7,7 @@ import {
7
7
  DyFM_Metadata,
8
8
  DyFM_sameObjects
9
9
  } from '@futdevpro/fsm-dynamo';
10
- import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
10
+ import { DyFM_OpenAI_Settings, DyFM_OpenAIEmbeddingModel, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
11
11
 
12
12
  import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
13
13
  import { DyNTS_DataService } from '../../../_services/base/data.service';
@@ -58,6 +58,7 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
58
58
  * (properties that are marked with vectorizeFrom in other properties)
59
59
  */
60
60
  readonly vectorizeProperties: DyFM_DataProperty_Params<any>[];
61
+ readonly vectorizedProperties: DyFM_DataProperty_Params<any>[];
61
62
  /* readonly vectorizeProperties: { from: string, to: string }[]; */
62
63
 
63
64
  embedding_CS: DyNTS_OAI_Embedding_ControlService;
@@ -81,11 +82,25 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
81
82
  issuer: string
82
83
  ) {
83
84
  super(data, dataParams, issuer);
85
+
86
+ this.vectorizedProperties = Object.values(dataParams.properties).filter(
87
+ (property: DyFM_DataProperty_Params<any>) => property.vectorizedFrom
88
+ );
89
+
90
+ this.vectorizeProperties = Object.keys(dataParams.properties).filter(
91
+ (key: string): boolean => this.vectorizedProperties.some(
92
+ (property: DyFM_DataProperty_Params<any>) => property.vectorizedFrom.includes(key)
93
+ )
94
+ ).map(
95
+ (key: string): DyFM_DataProperty_Params<any> => dataParams.properties[key] as DyFM_DataProperty_Params<any>
96
+ );
84
97
 
85
- const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
86
- (key: string): boolean => (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom &&
98
+ /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
99
+ (key: string): boolean => (
100
+ dataParams.properties[key] as DyFM_DataProperty_Params<any>
101
+ ).vectorizedFrom &&
87
102
  (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
88
- );
103
+ ); */
89
104
 
90
105
  if (!openAISettings.openAIConfig.apiKey) {
91
106
  throw new DyFM_Error({
@@ -96,7 +111,8 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
96
111
  });
97
112
  }
98
113
 
99
- const model = openAISettings.defaultSettings.useModel ?? DyFM_OpenAIModel.textEmbedding_3Large;
114
+ const model: DyFM_OpenAIEmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OpenAIEmbeddingModel ??
115
+ DyFM_OpenAIModel.textEmbedding_3Large;
100
116
 
101
117
  /* if (
102
118
  !DyFM_allOpenAIEmbeddingModels.includes(model)
@@ -109,7 +125,29 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
109
125
  });
110
126
  } */
111
127
 
112
- if (!vectorizePropertyKeys.length) {
128
+ if (!this.vectorizedProperties.length) {
129
+ throw new Error(
130
+ 'vectorizedProperties is missing! ' +
131
+ `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter(
132
+ (key: string): boolean => Boolean(
133
+ (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
134
+ )
135
+ ).map(key => `"${key}"`).join(', ')}`
136
+ );
137
+ }
138
+
139
+ if (!this.vectorizeProperties.length) {
140
+ throw new Error(
141
+ 'vectorizeProperties is missing! ' +
142
+ `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
143
+ (key: string): boolean => Boolean(
144
+ (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
145
+ )
146
+ ).map(key => `"${key}"`).join(', ')}`
147
+ );
148
+ }
149
+
150
+ /* if (!vectorizePropertyKeys.length) {
113
151
  throw new Error(
114
152
  'vectorizedPropertyKeys is missing! ' +
115
153
  `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
@@ -118,16 +156,20 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
118
156
  )
119
157
  ).map(key => `"${key}"`).join(', ')}`
120
158
  );
121
- }
159
+ } */
122
160
 
123
- vectorizePropertyKeys.forEach((key: string): void => {
161
+ /* vectorizePropertyKeys.forEach((key: string): void => {
124
162
  dataParams.properties[key].embeddingModel ??= model;
163
+ }); */
164
+
165
+ this.vectorizeProperties.forEach((property: DyFM_DataProperty_Params<any>): void => {
166
+ property.embeddingModel ??= model;
125
167
  });
126
168
 
127
- this.vectorizeProperties = vectorizePropertyKeys.map(
169
+ /* this.vectorizeProperties = vectorizePropertyKeys.map(
128
170
  (key: string): DyFM_DataProperty_Params<any> =>
129
171
  dataParams.properties[key] as DyFM_DataProperty_Params<any>
130
- );
172
+ ); */
131
173
  }
132
174
 
133
175
  /**
@@ -140,43 +182,22 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
140
182
  *
141
183
  * @returns The vectorized data
142
184
  */
143
- async vectorizeDataProperties(data?: T, alwaysVectorize?: boolean): Promise<T> {
185
+ async vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T> {
144
186
  try {
145
187
  let existingData: T;
146
188
 
147
- if (!alwaysVectorize) {
148
- existingData = await this.getDataById(data._id, true, true);
189
+ if (!forceVectorize) {
190
+ const existingData: T = await this.getDataById(newData._id, true, true);
149
191
  }
150
192
 
151
193
  await DyFM_Array.asyncForEachAllAtOnce(
152
- this.vectorizeProperties,
194
+ this.vectorizedProperties,
153
195
  async (property: DyFM_DataProperty_Params<any>) => {
154
- if (
155
- data[property.vectorizedFrom] &&
156
- (!data[property.key]?.length || alwaysVectorize || property.vectorizeAlways ||
157
- !DyFM_sameObjects(existingData[property.key], data[property.key])
158
- )
159
- ) {
160
- if (!property.embeddingModel) {
161
- throw new DyFM_Error({
162
- ...this.getDefaultErrorSettings(
163
- 'vectorizeDataProperties',
164
- new Error(`Property "${property.key}" has no embedding model!`)
165
- ),
166
-
167
- errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
168
- });
169
- }
170
-
171
- data[property.key] = await this.vectorize(
172
- data[property.vectorizedFrom],
173
- property.embeddingModel
174
- );
175
- }
196
+ await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
176
197
  }
177
198
  );
178
199
 
179
- return data;
200
+ return newData;
180
201
  } catch (error) {
181
202
  throw new DyFM_Error({
182
203
  ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
@@ -184,6 +205,72 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
184
205
  }
185
206
  }
186
207
 
208
+ private async checkAndVectorizeSingleProperty(
209
+ vectorizedProperty: DyFM_DataProperty_Params<any>,
210
+ newData: T,
211
+ existingData: T,
212
+ forceVectorize?: boolean
213
+ ): Promise<void> {
214
+ const newValue: string = await this.getVectorizingDataForVectorizedProperty(
215
+ vectorizedProperty,
216
+ newData
217
+ );
218
+ const existingValue: string = await this.getVectorizingDataForVectorizedProperty(
219
+ vectorizedProperty,
220
+ existingData
221
+ );
222
+
223
+ const vectorizedFrom: string = vectorizedProperty.vectorizedFrom[0];
224
+ const vectorizingPropertyHaveValue: boolean = Boolean(newData[vectorizedFrom]);
225
+ const newDataIsNewChanged: boolean =
226
+ (!existingValue && vectorizingPropertyHaveValue) ||
227
+ !DyFM_sameObjects(existingValue, newValue);
228
+
229
+ if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
230
+ if (!vectorizedProperty.embeddingModel) {
231
+ throw new DyFM_Error({
232
+ ...this.getDefaultErrorSettings(
233
+ 'vectorizeDataProperties',
234
+ new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)
235
+ ),
236
+
237
+ errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
238
+ });
239
+ }
240
+
241
+ newData[vectorizedProperty.key] = await this.vectorize(
242
+ newValue,
243
+ vectorizedProperty.embeddingModel
244
+ );
245
+ }
246
+ }
247
+
248
+ private async getVectorizingDataForVectorizedProperty(
249
+ vectorizedProperty: DyFM_DataProperty_Params<any>,
250
+ data: T,
251
+ /* alwaysVectorize?: boolean */
252
+ ): Promise<string> {
253
+ if (vectorizedProperty.vectorizedFrom.length === 1) {
254
+ const value: unknown = data[vectorizedProperty.vectorizedFrom[0]];
255
+
256
+ if (typeof value === 'string') {
257
+ return value;
258
+ } else if (Array.isArray(value) || typeof value === 'object') {
259
+ return JSON.stringify(value);
260
+ } else {
261
+ return `${value}`;
262
+ }
263
+ } else {
264
+ const object: any = {};
265
+
266
+ vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
267
+ object[key] = data[key];
268
+ });
269
+
270
+ return JSON.stringify(object);
271
+ }
272
+ }
273
+
187
274
  /**
188
275
  * Vectorizes the input
189
276
  *
@@ -23,6 +23,7 @@ export * from './_collections/oai-chunking.util';
23
23
  export * from './_collections/oai-document.util';
24
24
  export * from './_collections/oai-global-settings.const';
25
25
  export * from './_collections/oai-llm-predefined-requests.conts';
26
+ export * from './_collections/oai-llm-predefined-requests-hu.conts';
26
27
 
27
28
  // ENUMS
28
29
  export * from './_enums/oai-compare-result-type.enum';
@@ -32,6 +33,7 @@ export * from './_enums/oai-gpt-message-role.enum';
32
33
  export * from './_models/oai-doc-chunk.data-model';
33
34
  // models/INTERFACES
34
35
  export * from './_models/interfaces/oai-chunk-compare-result.interface';
36
+ export * from './_models/interfaces/oai-page-compare-result.interface';
35
37
  export * from './_models/interfaces/oai-document-page.interface';
36
38
  export * from './_models/interfaces/oai-global-settings.interface';
37
39
  export * from './_models/interfaces/oai-gpt-message.interface';
@@ -549,6 +549,8 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
549
549
  * @example
550
550
  * // or by id that is in list:
551
551
  * { userIds: { $in: this.userId } }
552
+ * // or by userIds:
553
+ * { userId: { $in: userIds } }
552
554
  * //
553
555
  * @example
554
556
  * // or by number or Date that is Greater Than AND Less Than:
@@ -605,6 +607,8 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
605
607
  * @example
606
608
  * // or by id that is in list:
607
609
  * { userIds: { $in: this.userId } }
610
+ * // or by userIds:
611
+ * { userId: { $in: userIds } }
608
612
  * //
609
613
  * @example
610
614
  * // or by number or Date that is Greater Than AND Less Than:
@@ -661,6 +665,8 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
661
665
  * @example
662
666
  * // or by id that is in list:
663
667
  * { userIds: { $in: this.userId } }
668
+ * // or by userIds:
669
+ * { userId: { $in: userIds } }
664
670
  * //
665
671
  * @example
666
672
  * // or by number or Date that is Greater Than AND Less Than:
@@ -735,6 +735,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
735
735
  * @example
736
736
  * // or by id that is in list:
737
737
  * { userIds: { $in: this.userId } }
738
+ * // or by userIds:
739
+ * { userId: { $in: userIds } }
738
740
  * //
739
741
  * @example
740
742
  * // or by number or Date that is Greater Than AND Less Than:
@@ -778,6 +780,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
778
780
  * @example
779
781
  * // or by id that is in list:
780
782
  * { userIds: { $in: this.userId } }
783
+ * // or by userIds:
784
+ * { userId: { $in: userIds } }
781
785
  * //
782
786
  * @example
783
787
  * // or by number or Date that is Greater Than AND Less Than:
@@ -825,6 +829,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
825
829
  * @example
826
830
  * // or by id that is in list:
827
831
  * { userIds: { $in: this.userId } }
832
+ * // or by userIds:
833
+ * { userId: { $in: userIds } }
828
834
  * //
829
835
  * @example
830
836
  * // or by number or Date that is Greater Than AND Less Than:
@@ -929,6 +935,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
929
935
  * @example
930
936
  * // or by id that is in list:
931
937
  * { userIds: { $in: this.userId } }
938
+ * // or by userIds:
939
+ * { userId: { $in: userIds } }
932
940
  * //
933
941
  * @example
934
942
  * // or by number or Date that is Greater Than AND Less Than:
@@ -1008,6 +1016,8 @@ export class DyNTS_DBService<T extends DyFM_Metadata> {
1008
1016
  * @example
1009
1017
  * // or by id that is in list:
1010
1018
  * { userIds: { $in: this.userId } }
1019
+ * // or by userIds:
1020
+ * { userId: { $in: userIds } }
1011
1021
  * //
1012
1022
  * @example
1013
1023
  * // or by number or Date that is Greater Than AND Less Than: