@futdevpro/nts-dynamo 1.11.2 → 1.11.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.d.ts +1 -2
  2. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.d.ts.map +1 -1
  3. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.js +16 -10
  4. package/build/_modules/discord-assistant/_models/dias-knowledge.data-model.js.map +1 -1
  5. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts +9 -5
  6. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts.map +1 -1
  7. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js +18 -6
  8. package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js.map +1 -1
  9. package/build/_modules/open-ai/_collections/oai-chunking.util.d.ts +3 -3
  10. package/build/_modules/open-ai/_collections/oai-chunking.util.d.ts.map +1 -1
  11. package/build/_modules/open-ai/_collections/oai-chunking.util.js +36 -24
  12. package/build/_modules/open-ai/_collections/oai-chunking.util.js.map +1 -1
  13. package/build/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.d.ts +3 -3
  14. package/build/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.d.ts.map +1 -1
  15. package/build/_modules/open-ai/_models/interfaces/oai-document-page.interface.d.ts +9 -10
  16. package/build/_modules/open-ai/_models/interfaces/oai-document-page.interface.d.ts.map +1 -1
  17. package/build/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.d.ts +7 -9
  18. package/build/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.d.ts.map +1 -1
  19. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.d.ts +25 -15
  20. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.d.ts.map +1 -1
  21. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.js +73 -65
  22. package/build/_modules/open-ai/_models/oai-doc-chunk.data-model.js.map +1 -1
  23. package/build/_modules/open-ai/_services/oai-chunk.service-base.d.ts +10 -9
  24. package/build/_modules/open-ai/_services/oai-chunk.service-base.d.ts.map +1 -1
  25. package/build/_modules/open-ai/_services/oai-chunk.service-base.js +18 -33
  26. package/build/_modules/open-ai/_services/oai-chunk.service-base.js.map +1 -1
  27. package/build/_modules/open-ai/_services/oai-vector-data.service.d.ts +4 -1
  28. package/build/_modules/open-ai/_services/oai-vector-data.service.d.ts.map +1 -1
  29. package/build/_modules/open-ai/_services/oai-vector-data.service.js +80 -24
  30. package/build/_modules/open-ai/_services/oai-vector-data.service.js.map +1 -1
  31. package/build/_modules/open-ai/index.d.ts +1 -0
  32. package/build/_modules/open-ai/index.d.ts.map +1 -1
  33. package/build/_modules/open-ai/index.js +1 -0
  34. package/build/_modules/open-ai/index.js.map +1 -1
  35. package/package.json +3 -3
  36. package/src/_modules/discord-assistant/_models/dias-knowledge.data-model.ts +12 -5
  37. package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +33 -16
  38. package/src/_modules/open-ai/_collections/oai-chunking.util.ts +54 -35
  39. package/src/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.ts +8 -3
  40. package/src/_modules/open-ai/_models/interfaces/oai-document-page.interface.ts +16 -10
  41. package/src/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.ts +10 -13
  42. package/src/_modules/open-ai/_models/oai-doc-chunk.data-model.ts +81 -72
  43. package/src/_modules/open-ai/_services/oai-chunk.service-base.ts +36 -39
  44. package/src/_modules/open-ai/_services/oai-vector-data.service.ts +124 -37
  45. package/src/_modules/open-ai/index.ts +1 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@futdevpro/nts-dynamo",
3
- "version": "01.11.2",
3
+ "version": "01.11.5",
4
4
  "description": "Dynamic NodeTS (NodeJS-Typescript), MongoDB Backend System Framework by Future Development Program Ltd.",
5
5
  "DyBu_settings": {
6
6
  "packageType": "server-package",
@@ -163,12 +163,12 @@
163
163
  },
164
164
  "homepage": "https:/futdevpro.hu/projects/dynamo",
165
165
  "DISABLED": {
166
- "@futdevpro/fsm-dynamo": "file:../tgz-collection/dynamo-fsm/futdevpro-fsm-dynamo-01.11.2.tgz",
166
+ "@futdevpro/fsm-dynamo": "file:../tgz-collection/dynamo-fsm/futdevpro-fsm-dynamo-01.11.5.tgz",
167
167
 
168
168
  "empty": ""
169
169
  },
170
170
  "peerDependencies": {
171
- "@futdevpro/fsm-dynamo": "1.11.2",
171
+ "@futdevpro/fsm-dynamo": "1.11.5",
172
172
 
173
173
  "@types/express": "4.17.21",
174
174
  "@types/geoip-lite": "~1.4.1",
@@ -5,10 +5,12 @@ import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
5
5
  export class DyNTS_DiAs_Knowledge extends DyFM_Metadata {
6
6
 
7
7
  subject?: string;
8
- subjectVectorized?: number[] = [];
8
+ /* subjectVectorized?: number[] = []; */
9
9
 
10
10
  description?: string;
11
- descriptionVectorized?: number[] = [];
11
+ /* descriptionVectorized?: number[] = []; */
12
+
13
+ contentVectorized?: number[] = [];
12
14
 
13
15
  constructor(
14
16
  set?: DyNTS_DiAs_Knowledge,
@@ -28,19 +30,24 @@ export const DyNTS_DiAs_knowledge_dataParams = new DyFM_DataModel_Params<DyNTS_D
28
30
  type: 'string', index: true,
29
31
  required: true, unique: true,
30
32
  },
31
- subjectVectorized: {
33
+ /* subjectVectorized: {
32
34
  type: 'number[]',
33
35
  vectorizedFrom: 'subject',
34
36
  vectorizeUseIndex: 'subject',
35
- },
37
+ }, */
36
38
  description: {
37
39
  type: 'string', index: true,
38
40
  required: true,
39
41
  },
40
- descriptionVectorized: {
42
+ /* descriptionVectorized: {
41
43
  type: 'number[]',
42
44
  vectorizedFrom: 'description',
43
45
  vectorizeUseIndex: 'description',
46
+ }, */
47
+ contentVectorized: {
48
+ type: 'number[]',
49
+ vectorizedFrom: [ 'description', 'subject' ],
50
+ vectorizeUseIndex: 'contentVectorized',
44
51
  },
45
52
  },
46
53
  });
@@ -1,16 +1,20 @@
1
-
2
- import { DyNTS_OAI_DocChunk } from '../../open-ai/_models/oai-doc-chunk.data-model';
3
- import { DyNTS_OAI_Chunk_ServiceBase } from '../../open-ai/_services/oai-chunk.service-base';
4
- import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
1
+ import { DyNTS_OAI_Chunk, DyNTS_OAI_defaultChunk_dataParams } from '../../open-ai/_models/oai-doc-chunk.data-model';
2
+ import { DyNTS_OAI_Chunk_DataServiceBase } from '../../open-ai/_services/oai-chunk.service-base';
3
+ import { DyFM_DataModel_Params, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
5
4
  import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
6
5
  import { DyNTS_DiAs_Main_ControlService } from './dias-main.control-service';
7
6
  import { DyNTS_OAI_LLMChat_ServiceBase } from '../../open-ai/_services/oai-llm-chat.service-base';
8
7
  import { DyNTS_OAI_GPT_Message } from '../../open-ai/_models/interfaces/oai-gpt-message.interface';
9
8
  import { DyNTS_OAI_LLMDefaultPredefined_Requests } from '../../open-ai/_collections/oai-llm-predefined-requests.conts';
10
9
  import { DyNTS_OAI_Document_Util } from '../../open-ai/_collections/oai-document.util';
10
+ import { DyNTS_OAI_DocumentPage } from '../../open-ai/_models/interfaces/oai-document-page.interface';
11
+ import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
11
12
 
12
13
 
13
- export abstract class DyNTS_DiAs_Chunk_DataService extends DyNTS_OAI_Chunk_ServiceBase {
14
+ export abstract class DyNTS_DiAs_Chunk_DataService<
15
+ T_Chunk extends DyNTS_OAI_Chunk = DyNTS_OAI_Chunk,
16
+ T_Page extends DyNTS_OAI_DocumentPage<T_Chunk> = DyNTS_OAI_DocumentPage<T_Chunk>,
17
+ > extends DyNTS_OAI_Chunk_DataServiceBase<T_Chunk, T_Page> {
14
18
 
15
19
  protected abstract readonly mainDiscordBot_CS: DyNTS_DiAs_Main_ControlService;
16
20
 
@@ -20,10 +24,28 @@ export abstract class DyNTS_DiAs_Chunk_DataService extends DyNTS_OAI_Chunk_Servi
20
24
 
21
25
  constructor(
22
26
  issuer: string,
27
+ dataParams?: DyFM_DataModel_Params<T_Chunk>,
28
+ openAiSettings?: DyFM_OpenAI_Settings,
23
29
  ) {
24
- super(issuer);
30
+ super(
31
+ dataParams ?? DyNTS_OAI_defaultChunk_dataParams as DyFM_DataModel_Params<T_Chunk>,
32
+ openAiSettings ??
33
+ new DyFM_OpenAI_Settings({
34
+ openAIConfig: {
35
+ apiKey: DyNTS_global_settings.env_settings.openAi.apiKey,
36
+ organization: DyNTS_global_settings.env_settings.openAi.organization,
37
+ project: DyNTS_global_settings.env_settings.openAi.project,
38
+ },
39
+ defaultSettings: {
40
+ useModel: DyFM_OpenAIModel.textEmbedding_3Large,
41
+ },
42
+ }),
43
+ issuer
44
+ );
25
45
  }
26
46
 
47
+ abstract getPageLink(page: T_Page, issuer: string): string;
48
+
27
49
  async resolveConversation(
28
50
  conversation: DyNTS_OAI_GPT_Message[],
29
51
  ): Promise<string> {
@@ -71,7 +93,7 @@ export abstract class DyNTS_DiAs_Chunk_DataService extends DyNTS_OAI_Chunk_Servi
71
93
  numberOfCandidates: number = 100,
72
94
  ): Promise<string> {
73
95
  try {
74
- const searchResults: DyNTS_OAI_DocChunk[] = await this.vectorSearchUsingConversation(
96
+ const searchResults: T_Chunk[] = await this.vectorSearchUsingConversation(
75
97
  conversation,
76
98
  limit,
77
99
  numberOfCandidates,
@@ -79,13 +101,8 @@ export abstract class DyNTS_DiAs_Chunk_DataService extends DyNTS_OAI_Chunk_Servi
79
101
 
80
102
  const parsedSearchResults: string = searchResults.map((result) => {
81
103
  return '' +
82
- '\npage: ' + result.documentName +
83
- '\npageLink: ' + DyNTS_OAI_Document_Util.getClickupPageLink(
84
- result.documentName,
85
- result.workspaceId,
86
- result.documentId,
87
- result.documentPageId
88
- ) +
104
+ '\npage: ' + result.pageName +
105
+ '\npageLink: ' + result.pageLink +
89
106
  '\ncontent: ' +
90
107
  `\n${result.chunkParentedContent.replace(/\n/g, '\n ')}`
91
108
  }).join('\n\n');
@@ -117,7 +134,7 @@ export abstract class DyNTS_DiAs_Chunk_DataService extends DyNTS_OAI_Chunk_Servi
117
134
  conversation: DyNTS_OAI_GPT_Message[],
118
135
  limit: number = 3,
119
136
  numberOfCandidates: number = 100,
120
- ): Promise<DyNTS_OAI_DocChunk[]> {
137
+ ): Promise<T_Chunk[]> {
121
138
  try {
122
139
  const lastMessage: DyNTS_OAI_GPT_Message = conversation[conversation.length - 1];
123
140
 
@@ -134,7 +151,7 @@ export abstract class DyNTS_DiAs_Chunk_DataService extends DyNTS_OAI_Chunk_Servi
134
151
 
135
152
  DyFM_Log.testInfo('🔍 Document search question:', searchQuestion);
136
153
 
137
- const searchResults: DyNTS_OAI_DocChunk[] = await this.vectorSearch(
154
+ const searchResults: T_Chunk[] = await this.vectorSearch(
138
155
  searchQuestion + ' ' + lastMessage.content,
139
156
  'chunkParentedContentVectorized',
140
157
  limit,
@@ -1,5 +1,5 @@
1
1
  import { DyFM_AnyError, DyFM_Error, DyFM_Error_Settings, DyFM_Log } from '@futdevpro/fsm-dynamo';
2
- import { DyNTS_OAI_DocChunk } from '../_models/oai-doc-chunk.data-model';
2
+ import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
3
3
  import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
4
4
  import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
5
5
 
@@ -20,10 +20,14 @@ export class DyNTS_OAI_Chunking_Util {
20
20
  * @param issuer - Kérés kezdeményezője
21
21
  * @returns Promise<CU_DocChunk[]> - Létrehozott chunk-ok listája
22
22
  */
23
- static async chunkMdContent(
24
- page: DyNTS_OAI_DocumentPage,
23
+ static async chunkMdContent<
24
+ T_Chunk extends DyNTS_OAI_Chunk,
25
+ T_Page extends DyNTS_OAI_DocumentPage<T_Chunk>
26
+ >(
27
+ page: T_Page,
28
+ getPageLink: (page: T_Page, issuer: string) => string,
25
29
  issuer: string
26
- ): Promise<DyNTS_OAI_DocChunk[]> {
30
+ ): Promise<T_Chunk[]> {
27
31
  try {
28
32
  if (page.content.includes('#####')) {
29
33
  DyFM_Log.warn('lvl5+ headers are not supported yet for page: ' + page.name);
@@ -34,7 +38,7 @@ export class DyNTS_OAI_Chunking_Util {
34
38
  let leftovers = page.content;
35
39
  const maxChunkSize = DyNTS_global_settings.docChunking.maxChunkSize;
36
40
  const maxChunkCount = DyNTS_global_settings.docChunking.maxChunkCount;
37
- const chunks: DyNTS_OAI_DocChunk[] = [];
41
+ const chunks: T_Chunk[] = [];
38
42
  let chunkIndex = 0;
39
43
  let currentPosition = 0; // Track current position in original content
40
44
 
@@ -58,20 +62,25 @@ export class DyNTS_OAI_Chunking_Util {
58
62
  if (chunkContent.length > 0) {
59
63
  const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
60
64
  const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
65
+ const headParents = headerHierarchy.map(header => header.header);
66
+ const allFlaggedParents = [ ...page.allFlaggedParents, ...headParents ];
61
67
 
62
- const chunk = new DyNTS_OAI_DocChunk({
63
- workspaceId: page.workspaceId,
64
- documentId: page.docId,
65
- documentPageId: page._id,
68
+ const chunk: T_Chunk = new DyNTS_OAI_Chunk({
69
+ filePathParents: page.path,
70
+ chunkHeadParents: headParents,
66
71
 
67
- documentName: page.name,
68
- documentPath: page.path,
72
+ allFlaggedParents: allFlaggedParents,
73
+ allFlaggedParentsMerged: allFlaggedParents.join('/'),
69
74
 
75
+ documentName: page.documentId,
76
+ pageName: page.name,
77
+ pageLink: getPageLink(page, issuer),
70
78
  chunkIndex: chunkIndex,
71
- chunkPath: chunkPath,
72
79
 
73
80
  chunkOriginalContent: chunkContent,
74
- });
81
+
82
+ chunkParentedContent: chunkWithHierarchy,
83
+ }) as T_Chunk;
75
84
  chunk.chunkParentedContent = this.assembleChunkWithHeaders(chunk, issuer);
76
85
  chunks.push(chunk);
77
86
  }
@@ -99,23 +108,28 @@ export class DyNTS_OAI_Chunking_Util {
99
108
 
100
109
  const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
101
110
  const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
111
+ const headParents = headerHierarchy.map(header => header.header);
112
+ const allFlaggedParents = [ ...page.allFlaggedParents, ...headParents ];
102
113
 
103
114
  /* console.log(`Chunk path:`, chunkPath);
104
115
  console.log(`Chunk with hierarchy preview:`, chunkWithHierarchy.substring(0, 100) + '...'); */
105
116
 
106
- const chunk = new DyNTS_OAI_DocChunk({
107
- workspaceId: page.workspaceId,
108
- documentId: page.docId,
109
- documentPageId: page._id,
110
-
111
- documentName: page.name,
112
- documentPath: page.path,
117
+ const chunk: T_Chunk = new DyNTS_OAI_Chunk({
118
+ filePathParents: page.path,
119
+ chunkHeadParents: headParents,
113
120
 
121
+ allFlaggedParents: allFlaggedParents,
122
+ allFlaggedParentsMerged: allFlaggedParents.join('/'),
123
+
124
+ documentName: page.documentId,
125
+ pageName: page.name,
126
+ pageLink: getPageLink(page, issuer),
114
127
  chunkIndex: chunkIndex,
115
- chunkPath: chunkPath,
116
-
128
+
117
129
  chunkOriginalContent: chunkContent,
118
- });
130
+
131
+ chunkParentedContent: chunkWithHierarchy,
132
+ }) as T_Chunk;
119
133
  chunk.chunkParentedContent = this.assembleChunkWithHeaders(chunk, issuer);
120
134
  chunks.push(chunk);
121
135
  chunkIndex++;
@@ -132,20 +146,25 @@ export class DyNTS_OAI_Chunking_Util {
132
146
  const chunkContent = leftovers.substring(0, maxChunkSize).trim();
133
147
  const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
134
148
  const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
149
+ const headParents = headerHierarchy.map(header => header.header);
150
+ const allFlaggedParents = [ ...page.allFlaggedParents, ...headParents ];
135
151
 
136
- const chunk = new DyNTS_OAI_DocChunk({
137
- workspaceId: page.workspaceId,
138
- documentId: page.docId,
139
- documentPageId: page._id,
152
+ const chunk: T_Chunk = new DyNTS_OAI_Chunk({
153
+ filePathParents: page.path,
154
+ chunkHeadParents: headParents,
140
155
 
141
- documentName: page.name,
142
- documentPath: page.path,
156
+ allFlaggedParents: allFlaggedParents,
157
+ allFlaggedParentsMerged: allFlaggedParents.join('/'),
143
158
 
159
+ documentName: page.documentId,
160
+ pageName: page.name,
161
+ pageLink: getPageLink(page, issuer),
144
162
  chunkIndex: chunkIndex,
145
- chunkPath: chunkPath,
146
163
 
147
164
  chunkOriginalContent: chunkContent,
148
- });
165
+
166
+ chunkParentedContent: chunkWithHierarchy,
167
+ }) as T_Chunk;
149
168
  chunk.chunkParentedContent = this.assembleChunkWithHeaders(chunk, issuer);
150
169
  chunks.push(chunk);
151
170
  chunkIndex++;
@@ -190,12 +209,12 @@ export class DyNTS_OAI_Chunking_Util {
190
209
  }
191
210
  }
192
211
 
193
- private static assembleChunkWithHeaders(chunk: DyNTS_OAI_DocChunk, issuer: string): string {
212
+ private static assembleChunkWithHeaders(chunk: DyNTS_OAI_Chunk, issuer: string): string {
194
213
  try {
195
- const fileHeaders = chunk.documentPath.map(
214
+ const fileHeaders = chunk.filePathParents.map(
196
215
  (header, index) => `${'>'.repeat(index + 1)} **${header}**`
197
216
  ).join('\n');
198
- const headers = chunk.chunkPath.filter(
217
+ const headers = chunk.chunkHeadParents.filter(
199
218
  path => !chunk.chunkOriginalContent.includes(path)
200
219
  ).join('\n');
201
220
  return `${fileHeaders}\n\n${headers}\n\n${chunk.chunkOriginalContent}`;
@@ -208,7 +227,7 @@ export class DyNTS_OAI_Chunking_Util {
208
227
  }
209
228
 
210
229
  static reassembleChunksToContent(
211
- chunks: DyNTS_OAI_DocChunk[],
230
+ chunks: DyNTS_OAI_Chunk[],
212
231
  issuer: string
213
232
  ): string {
214
233
  try {
@@ -1,10 +1,15 @@
1
1
 
2
2
 
3
3
  import { DyNTS_OAI_CompareResult_Type } from '../../_enums/oai-compare-result-type.enum';
4
- import { DyNTS_OAI_DocChunk } from '../oai-doc-chunk.data-model';
4
+ import { DyNTS_OAI_Chunk } from '../oai-doc-chunk.data-model';
5
5
 
6
6
 
7
- export interface DyNTS_OAI_ChunkCompareResult {
7
+ export interface DyNTS_OAI_ChunkCompareResult<
8
+ T_Chunk extends DyNTS_OAI_Chunk = DyNTS_OAI_Chunk
9
+ > {
8
10
  result: DyNTS_OAI_CompareResult_Type;
9
- oldChunk: DyNTS_OAI_DocChunk;
11
+ oldChunk: T_Chunk;
10
12
  }
13
+
14
+
15
+
@@ -1,21 +1,27 @@
1
1
  import { DyFM_Metadata } from '@futdevpro/fsm-dynamo';
2
- import { DyNTS_OAI_DocChunk } from '../oai-doc-chunk.data-model';
2
+ import { DyNTS_OAI_Chunk } from '../oai-doc-chunk.data-model';
3
3
 
4
4
 
5
5
 
6
6
 
7
- export interface DyNTS_OAI_DocumentPage extends DyFM_Metadata {
8
- workspaceId: string;
9
- docId: string;
10
- parentPageId: string;
7
+ export interface DyNTS_OAI_DocumentPage<
8
+ T_Chunk extends DyNTS_OAI_Chunk = DyNTS_OAI_Chunk
9
+ > {
10
+ documentId: string;
11
+ pageId: string;
11
12
 
13
+ name: string;
14
+
15
+ allFlaggedParents: string[];
16
+ allFlaggedParentsMerged: string;
17
+
12
18
  /** Path to the page in the document, starting with the document name */
13
19
  path: string[];
14
- documentParent: string;
15
20
 
16
- name: string;
21
+ chunks: T_Chunk[];
17
22
  content: string;
18
-
19
- pages: DyNTS_OAI_DocumentPage[];
20
- chunks: DyNTS_OAI_DocChunk[];
23
+
24
+ pages?: DyNTS_OAI_DocumentPage<T_Chunk>[];
21
25
  }
26
+
27
+
@@ -1,18 +1,15 @@
1
+ import { DyNTS_OAI_Chunk } from '../oai-doc-chunk.data-model';
2
+ import { DyNTS_OAI_ChunkCompareResult, DyNTS_OAI_CompareResult_Type, DyNTS_OAI_DocumentPage } from '../..';
1
3
 
2
- import { DyNTS_OAI_CompareResult_Type } from '../../_enums/oai-compare-result-type.enum';
3
- import { DyNTS_OAI_DocChunk } from '../oai-doc-chunk.data-model';
4
- import { DyNTS_OAI_ChunkCompareResult } from './oai-chunk-compare-result.interface';
5
- import { DyNTS_OAI_DocumentPage } from './oai-document-page.interface';
6
-
7
-
8
-
9
-
10
- export interface DyNTS_OAI_PageCompareResult {
4
+ export interface DyNTS_OAI_PageCompareResult<
5
+ T_Chunk extends DyNTS_OAI_Chunk = DyNTS_OAI_Chunk,
6
+ T_Doc extends DyNTS_OAI_DocumentPage<T_Chunk> = DyNTS_OAI_DocumentPage<T_Chunk>,
7
+ > {
11
8
  result: DyNTS_OAI_CompareResult_Type;
12
- subjectPage: DyNTS_OAI_DocumentPage;
9
+ subjectPage: T_Doc;
13
10
  oldPageContent: string;
14
- chunkCompareResults: DyNTS_OAI_ChunkCompareResult[];
11
+ chunkCompareResults: DyNTS_OAI_ChunkCompareResult<T_Chunk>[];
15
12
 
16
- newChunks: DyNTS_OAI_DocChunk[];
17
- oldChunks: DyNTS_OAI_DocChunk[];
13
+ newChunks: T_Chunk[];
14
+ oldChunks: T_Chunk[];
18
15
  }
@@ -1,92 +1,101 @@
1
1
 
2
2
 
3
3
 
4
- import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
4
+ import { DyFM_DataModel_Params, DyFM_DataProperties, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
5
5
  import { DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
6
6
 
7
+ export class DyNTS_OAI_Chunk extends DyFM_Metadata {
8
+ /** Path to the page where the chunk is located ending with the page name */
9
+ filePathParents: string[] = [];
10
+ /** Headers that are parents of the chunk */
11
+ chunkHeadParents: string[] = [];
7
12
 
13
+ /**
14
+ * All parents of the chunk, flagged, from the highest level to the lowest local level
15
+ * like: System:FDP, Project:Organizer, Side:Backend, Module:Chat, etc.
16
+ * */
17
+ allFlaggedParents: string[] = [];
18
+ /**
19
+ * All parents of the chunk, flagged, merged into a with '/' as separator
20
+ * */
21
+ allFlaggedParentsMerged: string;
22
+
23
+ documentName: string;
24
+ pageName: string;
25
+ pageLink: string;
26
+ /** Index of the chunk in the page */
27
+ chunkIndex: number;
8
28
 
9
- export class DyNTS_OAI_DocChunk extends DyFM_Metadata {
10
- workspaceId?: string;
11
- documentId?: string;
12
- documentPageId?: string;
13
- mainDocumentName?: string;
14
- documentName?: string;
15
- documentPath?: string[];
29
+ chunkOriginalContent: string;
16
30
 
17
- chunkIndex?: number;
18
- /** parent headers */
19
- chunkPath?: string[];
20
-
21
- chunkOriginalContent?: string;
22
-
23
- chunkParentedContent?: string;
24
- chunkParentedContentVectorized?: number[] = [];
31
+ chunkParentedContent: string;
32
+ chunkParentedContentVectorized?: number[];
25
33
 
26
34
  constructor(
27
- set?: DyNTS_OAI_DocChunk,
35
+ set: DyNTS_OAI_Chunk
28
36
  ) {
29
37
  super(set);
30
-
31
- if (set) {
32
- Object.assign(this, set);
33
- }
38
+
39
+ Object.assign(this, set);
34
40
  }
35
41
  }
36
42
 
37
- export const DyNTS_OAI_docChunk_dataParams = new DyFM_DataModel_Params<DyNTS_OAI_DocChunk>({
38
- dataName: 'oai_doc_chunk',
39
- properties: {
40
- workspaceId: {
41
- type: 'string', index: true,
42
- required: true,
43
- },
44
- documentId: {
45
- type: 'string', index: true,
46
- required: true,
47
- },
48
- documentPageId: {
49
- type: 'string', index: true,
50
- required: true,
51
- },
52
- mainDocumentName: {
53
- type: 'string',
54
- required: true,
55
- },
56
- documentName: {
57
- type: 'string',
58
- required: true,
59
- },
60
- documentPath: {
61
- type: 'string[]',
62
- required: true,
63
- },
43
+ export const chunk_propertyParams: DyFM_DataProperties<DyNTS_OAI_Chunk> = {
44
+ filePathParents: {
45
+ type: 'string[]',
46
+ required: true,
47
+ },
48
+ chunkHeadParents: {
49
+ type: 'string[]',
50
+ required: true,
51
+ },
52
+
53
+ allFlaggedParents: {
54
+ type: 'string[]',
55
+ required: true,
56
+ },
57
+ allFlaggedParentsMerged: {
58
+ type: 'string', index: true,
59
+ required: true,
60
+ },
64
61
 
65
- chunkIndex: {
66
- type: 'number', index: true,
67
- required: true,
68
- },
69
- chunkPath: {
70
- type: 'string[]',
71
- required: true,
72
- },
62
+ documentName: {
63
+ type: 'string', index: true,
64
+ required: true,
65
+ },
66
+ pageName: {
67
+ type: 'string', index: true,
68
+ required: true,
69
+ },
70
+ pageLink: {
71
+ type: 'string', index: true,
72
+ required: true,
73
+ },
74
+ chunkIndex: {
75
+ type: 'number', index: true,
76
+ required: true,
77
+ },
73
78
 
74
- chunkOriginalContent: {
75
- type: 'string',
76
- required: true,
77
- },
79
+ chunkOriginalContent: {
80
+ type: 'string',
81
+ required: true,
82
+ },
78
83
 
79
- chunkParentedContent: {
80
- type: 'string', index: true,
81
- required: true,
82
- },
83
- chunkParentedContentVectorized: {
84
- type: 'number[]',
85
- required: true, index: true,
86
- embeddingModel: DyFM_OpenAIModel.textEmbedding_3Large,
87
- vectorizedFrom: 'chunkParentedContent',
88
- vectorizeUseIndex: 'chunkParentedContentVectorized',
89
- vectorizeAlways: true,
90
- },
84
+ chunkParentedContent: {
85
+ type: 'string', // index: true,
86
+ required: true,
87
+ },
88
+ chunkParentedContentVectorized: {
89
+ type: 'number[]',
90
+ required: true,
91
+ embeddingModel: DyFM_OpenAIModel.textEmbedding_3Large,
92
+ vectorizedFrom: [ 'chunkParentedContent' ],
93
+ vectorizeUseIndex: 'chunkParentedContentVectorized',
94
+ vectorizeAlways: true,
91
95
  },
96
+ }
97
+
98
+ export const DyNTS_OAI_defaultChunk_dataParams: DyFM_DataModel_Params<DyNTS_OAI_Chunk> = new DyFM_DataModel_Params<DyNTS_OAI_Chunk>({
99
+ dataName: 'doc_chunk',
100
+ properties: chunk_propertyParams,
92
101
  });