@futdevpro/nts-dynamo 1.14.28 → 1.14.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.d.ts +22 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.d.ts.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.js +187 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.js.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.d.ts +27 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.d.ts.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.js +277 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.js.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.d.ts +23 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.d.ts.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.js +319 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.js.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.d.ts +118 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.d.ts.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.js +460 -0
- package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.js.map +1 -0
- package/build/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.d.ts +9 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.d.ts.map +1 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.js +14 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.js.map +1 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.d.ts +49 -3
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.d.ts.map +1 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.js +72 -2
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.js.map +1 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.d.ts +61 -3
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.d.ts.map +1 -1
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.js +190 -4
- package/build/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.js.map +1 -1
- package/build/_modules/ai/_modules/open-ai/index.d.ts +4 -4
- package/build/_modules/ai/_modules/open-ai/index.d.ts.map +1 -1
- package/build/_modules/ai/_modules/open-ai/index.js +6 -5
- package/build/_modules/ai/_modules/open-ai/index.js.map +1 -1
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts +1 -1
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.d.ts.map +1 -1
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js +1 -1
- package/build/_modules/discord-assistant/_services/dias-chunk.data-service.js.map +1 -1
- package/package.json +1 -1
- package/src/_modules/ai/_modules/open-ai/_services/{oai-doc-chunk-data.service.ts → data-services/oai-doc-chunk-data.service.ts} +8 -8
- package/src/_modules/ai/_modules/open-ai/_services/{oai-doc-page.data-service.ts → data-services/oai-doc-page.data-service.ts} +8 -8
- package/src/_modules/ai/_modules/open-ai/_services/{oai-document.data-service.ts → data-services/oai-document.data-service.ts} +7 -7
- package/src/_modules/ai/_modules/open-ai/_services/{oai-vector-data.service.ts → data-services/oai-vector-data.service.ts} +4 -4
- package/src/_modules/ai/_modules/open-ai/_services/oai-embedding.control-service.ts +18 -2
- package/src/_modules/ai/_modules/open-ai/_services/oai-llm-chat.service-base.ts +84 -3
- package/src/_modules/ai/_modules/open-ai/_services/oai-llm.service-base.ts +240 -5
- package/src/_modules/ai/_modules/open-ai/index.ts +6 -5
- package/src/_modules/discord-assistant/_services/dias-chunk.data-service.ts +1 -1
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { DyFM_DataModel_Params } from '@futdevpro/fsm-dynamo';
|
|
2
|
+
import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
|
|
3
|
+
import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
|
|
4
|
+
import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
|
|
5
|
+
import { DyNTS_DAI_DocChunkCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
|
|
6
|
+
import { DyNTS_DAI_DocPage } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
|
|
7
|
+
import { DyNTS_DAI_Document } from '../../../document-ai/_models/data-models/dai-document.data-model';
|
|
8
|
+
import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
|
|
9
|
+
import { DyNTS_OAI_DocPageDataService } from './oai-doc-page.data-service';
|
|
10
|
+
export declare class DyNTS_OAI_DocChunkDataService<T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk, T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, T_ExternalDocumentId = string, T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>> extends DyNTS_OAI_VectorDataService<T_Chunk> {
|
|
11
|
+
constructor(issuer: string, openAiSettings: DyFM_OAI_Settings, dataParams?: DyFM_DataModel_Params<T_Chunk>);
|
|
12
|
+
getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>;
|
|
13
|
+
getDocPageDataService(): DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>;
|
|
14
|
+
processPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]>;
|
|
15
|
+
processNewPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]>;
|
|
16
|
+
compareChunk(newChunk: T_Chunk, oldChunk: T_Chunk): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>>;
|
|
17
|
+
private compareStringArrays;
|
|
18
|
+
chunkPage(page: T_Page): Promise<T_Chunk[]>;
|
|
19
|
+
clickupWorkspaceId: string;
|
|
20
|
+
getPageLink(page: T_Page): string;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=oai-doc-chunk-data.service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oai-doc-chunk-data.service.d.ts","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,qBAAqB,EAA6F,MAAM,uBAAuB,CAAC;AACrK,OAAO,EAAE,iBAAiB,EAAkB,MAAM,kCAAkC,CAAC;AACrF,OAAO,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AAExE,OAAO,EAAE,kBAAkB,EAAiC,MAAM,mEAAmE,CAAC;AACtI,OAAO,EAAE,+BAA+B,EAAE,MAAM,gFAAgF,CAAC;AAEjI,OAAO,EAAE,iBAAiB,EAAE,MAAM,kEAAkE,CAAC;AAErG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kEAAkE,CAAC;AACtG,OAAO,EAAE,6BAA6B,EAAE,MAAM,6BAA6B,CAAC;AAC5E,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAI3E,qBAAa,6BAA6B,CACxC,OAAO,SAAS,kBAAkB,GAAG,kBAAkB,EACvD,MAAM,SAAS,iBAAiB,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,EACtE,oBAAoB,GAAG,MAAM,EAC7B,UAAU,SAAS,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,GAAG,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,CACxI,SAAQ,2BAA2B,CAAC,OAAO,CAAC;gBAK1C,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,iBAAiB,EACjC,UAAU,GAAE,qBAAqB,CAAC,OAAO,CAAmE;IAU9G,sBAAsB,IAAI,6BAA6B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,UAAU,CAAC;IAO1G,qBAAqB,IAAI,4BAA4B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,UAAU,CAAC;IAOlG,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,+BAA+B,CAAC,OAAO,CAAC,EAAE,CAAC;IA8D9E,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,+BAA+B,CAAC,OAAO,CAAC,EAAE,CAAC;IA8BjF,YAAY,CAChB,QAAQ,EAAE,OAAO,EACjB,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC,+BAA+B,CAAC,OAAO,CAAC,CAAC;IA+CpD,OAAO,CAAC,mBAAmB;IAOrB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAuCjD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;CAkDlC"}
|
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.js
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DyNTS_OAI_DocChunkDataService = void 0;
|
|
4
|
+
const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
|
|
5
|
+
const oai_vector_data_service_1 = require("./oai-vector-data.service");
|
|
6
|
+
const dai_chunking_util_1 = require("../../../document-ai/_collections/dai-chunking.util");
|
|
7
|
+
const dai_doc_chunk_data_model_1 = require("../../../document-ai/_models/data-models/dai-doc-chunk.data-model");
|
|
8
|
+
const dai_compare_result_type_enum_1 = require("../../../document-ai/_enums/dai-compare-result-type.enum");
|
|
9
|
+
const oai_document_data_service_1 = require("./oai-document.data-service");
|
|
10
|
+
const oai_doc_page_data_service_1 = require("./oai-doc-page.data-service");
|
|
11
|
+
const dai_document_util_1 = require("../../../document-ai/_collections/dai-document.util");
|
|
12
|
+
class DyNTS_OAI_DocChunkDataService extends oai_vector_data_service_1.DyNTS_OAI_VectorDataService {
|
|
13
|
+
/* override debugLog: boolean = false; */
|
|
14
|
+
constructor(issuer, openAiSettings, dataParams = dai_doc_chunk_data_model_1.DyNTS_DAI_docChunk_dataParams) {
|
|
15
|
+
super(null, dataParams, openAiSettings, issuer);
|
|
16
|
+
}
|
|
17
|
+
getDocumentDataService() {
|
|
18
|
+
return new oai_document_data_service_1.DyNTS_OAI_DocumentDataService(this.issuer, this.openAISettings);
|
|
19
|
+
}
|
|
20
|
+
getDocPageDataService() {
|
|
21
|
+
return new oai_doc_page_data_service_1.DyNTS_OAI_DocPageDataService(this.issuer, this.openAISettings);
|
|
22
|
+
}
|
|
23
|
+
async processPage(page) {
|
|
24
|
+
try {
|
|
25
|
+
const newChunks = await this.chunkPage(page);
|
|
26
|
+
const oldChunks = await this.findDataList({
|
|
27
|
+
pageId: page._id,
|
|
28
|
+
});
|
|
29
|
+
oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
|
30
|
+
const chunkCompareResults = [];
|
|
31
|
+
for (let i = 0; i < newChunks.length; i++) {
|
|
32
|
+
const newChunk = newChunks[i];
|
|
33
|
+
newChunk.documentId = page.documentId;
|
|
34
|
+
newChunk.pageId = page._id;
|
|
35
|
+
newChunk.chunkIndex = i;
|
|
36
|
+
const oldChunk = oldChunks[i];
|
|
37
|
+
if (oldChunk) {
|
|
38
|
+
newChunk._id = oldChunk._id;
|
|
39
|
+
// Compare first to determine if we need to save
|
|
40
|
+
const chunkCompareResult = await this.compareChunk(newChunk, oldChunk);
|
|
41
|
+
if (chunkCompareResult.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
|
|
42
|
+
// No changes detected, reuse existing chunk without saving
|
|
43
|
+
chunkCompareResult.newChunk = oldChunk;
|
|
44
|
+
chunkCompareResults.push(chunkCompareResult);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
// Changes detected, save the updated chunk
|
|
48
|
+
const savedChunk = await this.saveData(newChunk, true);
|
|
49
|
+
chunkCompareResult.newChunk = savedChunk;
|
|
50
|
+
chunkCompareResults.push(chunkCompareResult);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
// New chunk, save it
|
|
55
|
+
const savedChunk = await this.saveData(newChunk, true);
|
|
56
|
+
chunkCompareResults.push({
|
|
57
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
|
|
58
|
+
newChunk: savedChunk,
|
|
59
|
+
oldChunk: null,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
for (let i = newChunks.length; i < oldChunks.length; i++) {
|
|
64
|
+
const oldChunk = oldChunks[i];
|
|
65
|
+
await this.deleteData(oldChunk._id);
|
|
66
|
+
chunkCompareResults.push({
|
|
67
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted,
|
|
68
|
+
newChunk: null,
|
|
69
|
+
oldChunk: oldChunk,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
return chunkCompareResults;
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
76
|
+
...this.getDefaultErrorSettings('processPage', error),
|
|
77
|
+
errorCode: 'DCH-CSB-CCS0',
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async processNewPage(page) {
|
|
82
|
+
try {
|
|
83
|
+
const chunks = await this.chunkPage(page);
|
|
84
|
+
const chunkCompareResults = [];
|
|
85
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
86
|
+
const chunk = chunks[i];
|
|
87
|
+
chunk.documentId = page.documentId;
|
|
88
|
+
chunk.pageId = page._id;
|
|
89
|
+
chunk.chunkIndex = i;
|
|
90
|
+
const savedChunk = await this.saveData(chunk, true);
|
|
91
|
+
chunkCompareResults.push({
|
|
92
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
|
|
93
|
+
newChunk: savedChunk,
|
|
94
|
+
oldChunk: null,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
return chunkCompareResults;
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
101
|
+
...this.getDefaultErrorSettings('processNewChunks', error),
|
|
102
|
+
errorCode: 'DCH-CSB-CCS0',
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async compareChunk(newChunk, oldChunk) {
|
|
107
|
+
try {
|
|
108
|
+
if (!oldChunk) {
|
|
109
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
110
|
+
...this.getDefaultErrorSettings('compareChunk', new Error('oldChunk is required')),
|
|
111
|
+
errorCode: 'DCH-CSB-CC1',
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
if (this.debugLog) {
|
|
115
|
+
fsm_dynamo_1.DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
|
|
116
|
+
}
|
|
117
|
+
// Compare all relevant fields for comprehensive change detection
|
|
118
|
+
const contentEqual = oldChunk.chunkContent === newChunk.chunkContent;
|
|
119
|
+
const pathEqual = oldChunk.path === newChunk.path;
|
|
120
|
+
const documentNameEqual = oldChunk.documentName === newChunk.documentName;
|
|
121
|
+
const pageNameEqual = oldChunk.pageName === newChunk.pageName;
|
|
122
|
+
const pageLinkEqual = oldChunk.pageLink === newChunk.pageLink;
|
|
123
|
+
// Compare allFlaggedParents arrays
|
|
124
|
+
const allFlaggedParentsEqual = this.compareStringArrays(oldChunk.allFlaggedParents || [], newChunk.allFlaggedParents || []);
|
|
125
|
+
if (contentEqual && pathEqual && documentNameEqual && pageNameEqual && pageLinkEqual && allFlaggedParentsEqual) {
|
|
126
|
+
return {
|
|
127
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal,
|
|
128
|
+
newChunk: newChunk,
|
|
129
|
+
oldChunk: oldChunk,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified,
|
|
134
|
+
newChunk: newChunk,
|
|
135
|
+
oldChunk: oldChunk,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
140
|
+
...this.getDefaultErrorSettings('compareChunk', error),
|
|
141
|
+
errorCode: 'DCH-CSB-CC0',
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
compareStringArrays(arr1, arr2) {
|
|
146
|
+
if (arr1.length !== arr2.length) {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
return arr1.every((item, index) => item === arr2[index]);
|
|
150
|
+
}
|
|
151
|
+
async chunkPage(page) {
|
|
152
|
+
try {
|
|
153
|
+
page.chunks = await dai_chunking_util_1.DyNTS_DAI_DocChunking_Util.chunkMdContent(page, (page) => this.getPageLink(page), this.issuer);
|
|
154
|
+
if (this.debugLog) {
|
|
155
|
+
fsm_dynamo_1.DyFM_Log.H_info(`Processing file: ${page.name}`, {
|
|
156
|
+
'Page name': page.name,
|
|
157
|
+
'Page filePath': page.filePath,
|
|
158
|
+
'Page parentedPath': page.parentedPath,
|
|
159
|
+
'Page content': page.content,
|
|
160
|
+
'Page parentFlags': '\n' + page.allFlaggedParents.join('\n') + '\n',
|
|
161
|
+
chunks: page.chunks.map(chunk => {
|
|
162
|
+
return {
|
|
163
|
+
'Chunk name': chunk.documentName,
|
|
164
|
+
'Chunk path': chunk.path,
|
|
165
|
+
'Chunk index': chunk.chunkIndex,
|
|
166
|
+
'Chunk content': chunk.chunkContent,
|
|
167
|
+
'Chunk parentFlags': '\n' + chunk.allFlaggedParents.join('\n') + '\n',
|
|
168
|
+
};
|
|
169
|
+
})
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
return page.chunks;
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
176
|
+
...this.getDefaultErrorSettings('chunkDocPages', error),
|
|
177
|
+
errorCode: 'CCAP-CUD-CDP0',
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
clickupWorkspaceId;
|
|
182
|
+
getPageLink(page) {
|
|
183
|
+
return dai_document_util_1.DyNTS_DAI_Document_Util.getClickupPageLink(page.name, this.clickupWorkspaceId, page.documentId, page.externalPageId);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
exports.DyNTS_OAI_DocChunkDataService = DyNTS_OAI_DocChunkDataService;
|
|
187
|
+
//# sourceMappingURL=oai-doc-chunk-data.service.js.map
|
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oai-doc-chunk-data.service.js","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-chunk-data.service.ts"],"names":[],"mappings":";;;AAAA,sDAAqK;AAErK,uEAAwE;AACxE,2FAAiG;AACjG,gHAAsI;AAEtI,2GAAwG;AAIxG,2EAA4E;AAC5E,2EAA2E;AAC3E,2FAA8F;AAG9F,MAAa,6BAKX,SAAQ,qDAAoC;IAE5C,yCAAyC;IAEzC,YACE,MAAc,EACd,cAAiC,EACjC,aAA6C,wDAA+D;QAE5G,KAAK,CACH,IAAI,EACJ,UAAU,EACV,cAAc,EACd,MAAM,CACP,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,yDAA6B,CACtC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,cAAc,CACpB,CAAC;IACJ,CAAC;IAED,qBAAqB;QACnB,OAAO,IAAI,wDAA4B,CACrC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,cAAc,CACpB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAY;QAC5B,IAAI,CAAC;YACH,MAAM,SAAS,GAAc,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAExD,MAAM,SAAS,GAAc,MAAM,IAAI,CAAC,YAAY,CAAC;gBACnD,MAAM,EAAE,IAAI,CAAC,GAAG;aACe,CAAC,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;YAEtD,MAAM,mBAAmB,GAA+C,EAAE,CAAC;YAC3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9B,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;gBACtC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC;gBAC3B,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC;gBAExB,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;oBAC5B,gDAAgD;oBAChD,MAAM,kBAAkB,GAA6C,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAEjH,IAAI,kBAAkB,CAAC,MAAM,KAAK,2DAA4B,CAAC,KAAK,EAAE,CAAC;wBACrE,2DAA2D;wBAC3D,kBAAkB,CAAC,QAAQ,GAAG,QAAQ,CAAC;wBACvC,mBAAmB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAC/C,CAAC;yBAAM,CAAC;wBACN,2CAA2C;wBAC3C,MAAM,UAAU,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;wBAChE,kBAAkB,CAAC,QAAQ,GAAG,UAAU,CAAC;wBACzC,mBAAmB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAC/C,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,qBAAqB;oBACrB,MAAM,UAAU,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;oBAChE,mBAAmB,CAAC,IAAI,CAAC;wBACvB,MAAM,EAAE,2DAA4B,CAAC,GAAG;wBACxC,QAAQ,EAAE,UAAU;wBACpB,QAAQ,EAAE,IAAI;qBACf,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACpC,mBAAmB,CAAC,IAAI,CAAC;oBACvB,MAAM,EAAE,2DAA4B,CAAC,OAAO;oBAC5C,QAAQ,EAAE,IAAI;oBACd,QAAQ,EAAE,QAAQ;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,KAAK,CAAC;gBACrD,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAY;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAc,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAErD,MAAM,mBAAmB,GAA+C,EAAE,CAAC;YAE3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;gBACnC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC;gBACxB,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;gBAErB,MAAM,UAAU,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBAE7D,mBAAmB,CAAC,IAAI,CAAC;oBACvB,MAAM,EAAE,2DAA4B,CAAC,GAAG;oBACxC,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;YACL,CAAC;YAED,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,EAAE,KAAK,CAAC;gBAC1D,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,QAAiB,EACjB,QAAiB;QAEjB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,uBAAU,CAAC;oBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,cAAc,EAAE,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;oBAClF,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,qBAAQ,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,CAAC,QAAQ,WAAW,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC1F,CAAC;YAED,iEAAiE;YACjE,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,CAAC;YACrE,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC;YAClD,MAAM,iBAAiB,GAAG,QAAQ,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,CAAC;YAC1E,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAAC;YAC9D,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAAC;YAE9D,mCAAmC;YACnC,MAAM,sBAAsB,GAAG,IAAI,CAAC,mBAAmB,CACrD,QAAQ,CAAC,iBAAiB,IAAI,EAAE,EAChC,QAAQ,CAAC,iBAAiB,IAAI,EAAE,CACjC,CAAC;YAEF,IAAI,YAAY,IAAI,SAAS,IAAI,iBAAiB,IAAI,aAAa,IAAI,aAAa,IAAI,sBAAsB,EAAE,CAAC;gBAC/G,OAAO;oBACL,MAAM,EAAE,2DAA4B,CAAC,KAAK;oBAC1C,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,QAAQ;iBACnB,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,2DAA4B,CAAC,QAAQ;gBAC7C,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,QAAQ;aACnB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,cAAc,EAAE,KAAK,CAAC;gBACtD,SAAS,EAAE,aAAa;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,IAAc,EAAE,IAAc;QACxD,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC1B,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,MAAM,8CAA0B,CAAC,cAAc,CAC3D,IAAI,EACJ,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EACxC,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,qBAAQ,CAAC,MAAM,CACb,oBAAoB,IAAI,CAAC,IAAI,EAAE,EAC/B;oBACE,WAAW,EAAE,IAAI,CAAC,IAAI;oBACtB,eAAe,EAAE,IAAI,CAAC,QAAQ;oBAC9B,mBAAmB,EAAE,IAAI,CAAC,YAAY;oBACtC,cAAc,EAAE,IAAI,CAAC,OAAO;oBAC5B,kBAAkB,EAAE,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;oBACnE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;wBAC9B,OAAO;4BACL,YAAY,EAAE,KAAK,CAAC,YAAY;4BAChC,YAAY,EAAE,KAAK,CAAC,IAAI;4BACxB,aAAa,EAAE,KAAK,CAAC,UAAU;4BAC/B,eAAe,EAAE,KAAK,CAAC,YAAY;4BACnC,mBAAmB,EAAE,IAAI,GAAG,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;yBACtE,CAAA;oBACH,CAAC,CAAC;iBACH,CACF,CAAC;YACJ,CAAC;YAED,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,eAAe,EAAE,KAAK,CAAC;gBACvD,SAAS,EAAE,eAAe;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,kBAAkB,CAAS;IAC3B,WAAW,CAAC,IAAY;QACtB,OAAO,2CAAuB,CAAC,kBAAkB,CAC/C,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,cAAc,CACpB,CAAC;IACJ,CAAC;CA2CF;AAnRD,sEAmRC"}
|
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { DyFM_DataModel_Params } from '@futdevpro/fsm-dynamo';
|
|
2
|
+
import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
|
|
3
|
+
import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
|
|
4
|
+
import { DyNTS_DAI_DocPageCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
|
|
5
|
+
import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
|
|
6
|
+
import { DyNTS_DAI_DocPage } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
|
|
7
|
+
import { DyNTS_DAI_Document } from '../../../document-ai/_models/data-models/dai-document.data-model';
|
|
8
|
+
import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
|
|
9
|
+
import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
|
|
10
|
+
export declare class DyNTS_OAI_DocPageDataService<T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk, T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, T_ExternalDocumentId = string, T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>> extends DyNTS_DataService<T_Page> {
|
|
11
|
+
private docChunkDataServiceOpenAiSettings;
|
|
12
|
+
debugLog: boolean;
|
|
13
|
+
absoluteDeletesOnProcess: boolean;
|
|
14
|
+
constructor(issuer: string, docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings, data?: T_Page, dataParams?: DyFM_DataModel_Params<T_Page>);
|
|
15
|
+
getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>;
|
|
16
|
+
getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>;
|
|
17
|
+
processPages(documentId: string, pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]>;
|
|
18
|
+
processPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>>;
|
|
19
|
+
processNewPages(pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]>;
|
|
20
|
+
processNewPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>>;
|
|
21
|
+
/**
|
|
22
|
+
* this is a deep compare, that will update chunks if needed
|
|
23
|
+
* */
|
|
24
|
+
comparePage(newPage: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>>;
|
|
25
|
+
private compareStringArrays;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=oai-doc-page.data-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oai-doc-page.data-service.d.ts","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,qBAAqB,EAAsF,MAAM,uBAAuB,CAAC;AAC9J,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAErE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,8BAA8B,EAAE,MAAM,+EAA+E,CAAC;AAE/H,OAAO,EAAE,kBAAkB,EAAE,MAAM,mEAAmE,CAAC;AAEvG,OAAO,EAAE,iBAAiB,EAAgC,MAAM,kEAAkE,CAAC;AACnI,OAAO,EAAE,kBAAkB,EAAiC,MAAM,kEAAkE,CAAC;AAErI,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,6BAA6B,EAAE,MAAM,6BAA6B,CAAC;AAI5E,qBAAa,4BAA4B,CACvC,OAAO,SAAS,kBAAkB,GAAG,kBAAkB,EACvD,MAAM,SAAS,iBAAiB,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,EACtE,oBAAoB,GAAG,MAAM,EAC7B,UAAU,SAAS,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,GAAG,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,CACxI,SAAQ,iBAAiB,CAAC,MAAM,CAAC;IAO/B,OAAO,CAAC,iCAAiC;IAL3C,QAAQ,EAAE,OAAO,CAAQ;IACzB,wBAAwB,EAAE,OAAO,CAAQ;gBAGvC,MAAM,EAAE,MAAM,EACN,iCAAiC,EAAE,iBAAiB,EAC5D,IAAI,CAAC,EAAE,MAAM,EACb,UAAU,GAAE,qBAAqB,CAAC,MAAM,CAAiE;IAS3G,sBAAsB,IAAI,6BAA6B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,UAAU,CAAC;IAO1G,sBAAsB,IAAI,6BAA6B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,UAAU,CAAC;IAOpG,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC;IAsD7G,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IA+GnF,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC;IAkB5F,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAwC5F;;SAEK;IACC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IA+E7G,OAAO,CAAC,mBAAmB;CAoN5B"}
|
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.js
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DyNTS_OAI_DocPageDataService = void 0;
|
|
4
|
+
const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
|
|
5
|
+
const data_service_1 = require("../../../../../../_services/base/data.service");
|
|
6
|
+
const dai_doc_page_data_model_1 = require("../../../document-ai/_models/data-models/dai-doc-page.data-model");
|
|
7
|
+
const oai_doc_chunk_data_service_1 = require("./oai-doc-chunk-data.service");
|
|
8
|
+
const oai_document_data_service_1 = require("./oai-document.data-service");
|
|
9
|
+
const dai_compare_result_type_enum_1 = require("../../../document-ai/_enums/dai-compare-result-type.enum");
|
|
10
|
+
class DyNTS_OAI_DocPageDataService extends data_service_1.DyNTS_DataService {
|
|
11
|
+
docChunkDataServiceOpenAiSettings;
|
|
12
|
+
debugLog = true;
|
|
13
|
+
absoluteDeletesOnProcess = true;
|
|
14
|
+
constructor(issuer, docChunkDataServiceOpenAiSettings, data, dataParams = dai_doc_page_data_model_1.DyNTS_DAI_docPage_dataParams) {
|
|
15
|
+
super(data, dataParams, issuer);
|
|
16
|
+
this.docChunkDataServiceOpenAiSettings = docChunkDataServiceOpenAiSettings;
|
|
17
|
+
}
|
|
18
|
+
getDocumentDataService() {
|
|
19
|
+
return new oai_document_data_service_1.DyNTS_OAI_DocumentDataService(this.issuer, this.docChunkDataServiceOpenAiSettings);
|
|
20
|
+
}
|
|
21
|
+
getDocChunkDataService() {
|
|
22
|
+
return new oai_doc_chunk_data_service_1.DyNTS_OAI_DocChunkDataService(this.issuer, this.docChunkDataServiceOpenAiSettings);
|
|
23
|
+
}
|
|
24
|
+
async processPages(documentId, pages) {
|
|
25
|
+
try {
|
|
26
|
+
const compareResults = [];
|
|
27
|
+
for (const page of pages) {
|
|
28
|
+
page.documentId = documentId;
|
|
29
|
+
compareResults.push(await this.processPage(page));
|
|
30
|
+
}
|
|
31
|
+
// All pages that are not in the list, and are in the database saved to the same document
|
|
32
|
+
const deletedPages = await this.findDataList({
|
|
33
|
+
documentId: documentId,
|
|
34
|
+
_id: {
|
|
35
|
+
$nin: pages.map(page => page._id),
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
const docChunk_DS = this.getDocChunkDataService();
|
|
39
|
+
for (const deletedPage of deletedPages) {
|
|
40
|
+
const messages = [];
|
|
41
|
+
const chunksToDelete = await docChunk_DS.findDataList({
|
|
42
|
+
documentId: documentId,
|
|
43
|
+
pageId: deletedPage._id,
|
|
44
|
+
});
|
|
45
|
+
for (const chunk of chunksToDelete) {
|
|
46
|
+
await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
|
|
47
|
+
}
|
|
48
|
+
messages.push(`${chunksToDelete.length} chunks deleted`);
|
|
49
|
+
await this.deleteData(deletedPage._id, this.absoluteDeletesOnProcess);
|
|
50
|
+
messages.unshift(`Page "${deletedPage.name}" deleted (chunks: ${chunksToDelete.length})`);
|
|
51
|
+
compareResults.push({
|
|
52
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted,
|
|
53
|
+
newPage: null,
|
|
54
|
+
oldPage: deletedPage,
|
|
55
|
+
chunkCompareResults: [],
|
|
56
|
+
reports: messages,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
return compareResults;
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
63
|
+
...this.getDefaultErrorSettings('processPages', error),
|
|
64
|
+
errorCode: 'DCH-CSB-PP0',
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async processPage(page) {
|
|
69
|
+
try {
|
|
70
|
+
let existingPage;
|
|
71
|
+
if (page._id) {
|
|
72
|
+
existingPage = await this.findData({
|
|
73
|
+
_id: page._id,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
if (!page.externalPageId) {
|
|
78
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
79
|
+
...this.getDefaultErrorSettings('processPage', new Error('externalPageId is required')),
|
|
80
|
+
errorCode: 'DCH-CSB-PP0',
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
existingPage = await this.findData({
|
|
84
|
+
documentId: page.documentId,
|
|
85
|
+
externalPageId: page.externalPageId,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
let pageCompareResult;
|
|
89
|
+
if (existingPage) {
|
|
90
|
+
// Compare first to determine if we need to save
|
|
91
|
+
pageCompareResult = await this.comparePage(page, existingPage);
|
|
92
|
+
// Always process chunks after page comparison, regardless of page equality
|
|
93
|
+
const docChunk_DS = this.getDocChunkDataService();
|
|
94
|
+
const chunksCompareResults = await docChunk_DS.processPage(existingPage);
|
|
95
|
+
pageCompareResult.chunkCompareResults = chunksCompareResults;
|
|
96
|
+
// Recalculate result based on both page metadata and chunk changes
|
|
97
|
+
if (pageCompareResult.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal &&
|
|
98
|
+
chunksCompareResults.some(r => r.result !== dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal)) {
|
|
99
|
+
pageCompareResult.result = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
100
|
+
pageCompareResult.reports.push('Chunks modified');
|
|
101
|
+
}
|
|
102
|
+
if (pageCompareResult.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
|
|
103
|
+
// No changes detected, reuse existing page without saving
|
|
104
|
+
pageCompareResult.newPage = existingPage;
|
|
105
|
+
pageCompareResult.oldPage = existingPage;
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
// Changes detected, save the updated page
|
|
109
|
+
let pageDataToSave;
|
|
110
|
+
pageDataToSave = existingPage;
|
|
111
|
+
Object.assign(pageDataToSave, page);
|
|
112
|
+
delete pageDataToSave.chunks;
|
|
113
|
+
const savedPage = await this.saveData(pageDataToSave, true);
|
|
114
|
+
page.chunks.forEach((chunk) => {
|
|
115
|
+
chunk.documentId = savedPage.documentId;
|
|
116
|
+
chunk.pageId = savedPage._id;
|
|
117
|
+
});
|
|
118
|
+
page._id = savedPage._id;
|
|
119
|
+
pageCompareResult.newPage = page;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
// New page, save it first to get ID
|
|
124
|
+
const pageDataToSave = {
|
|
125
|
+
...page,
|
|
126
|
+
};
|
|
127
|
+
delete pageDataToSave.chunks;
|
|
128
|
+
const savedPage = await this.saveData(pageDataToSave, true);
|
|
129
|
+
page.chunks.forEach((chunk) => {
|
|
130
|
+
chunk.documentId = savedPage.documentId;
|
|
131
|
+
chunk.pageId = savedPage._id;
|
|
132
|
+
});
|
|
133
|
+
page._id = savedPage._id;
|
|
134
|
+
pageCompareResult = {
|
|
135
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
|
|
136
|
+
newPage: page,
|
|
137
|
+
oldPage: null,
|
|
138
|
+
chunkCompareResults: [],
|
|
139
|
+
reports: [
|
|
140
|
+
`New page: ${page.name}`,
|
|
141
|
+
`${page.chunks.length} new chunks in ${page.name}`,
|
|
142
|
+
],
|
|
143
|
+
};
|
|
144
|
+
const docChunk_DS = this.getDocChunkDataService();
|
|
145
|
+
const chunksCompareResults = await docChunk_DS.processNewPage(savedPage);
|
|
146
|
+
pageCompareResult.chunkCompareResults = chunksCompareResults;
|
|
147
|
+
}
|
|
148
|
+
return pageCompareResult;
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
152
|
+
...this.getDefaultErrorSettings('processPage', error),
|
|
153
|
+
errorCode: 'DCH-CSB-PP0',
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
async processNewPages(pages) {
|
|
158
|
+
try {
|
|
159
|
+
const pageCompareResults = [];
|
|
160
|
+
for (const page of pages) {
|
|
161
|
+
const pageCompareResult = await this.processNewPage(page);
|
|
162
|
+
pageCompareResults.push(pageCompareResult);
|
|
163
|
+
}
|
|
164
|
+
return pageCompareResults;
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
168
|
+
...this.getDefaultErrorSettings('processNewPages', error),
|
|
169
|
+
errorCode: 'DCH-CSB-PP0',
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
async processNewPage(page) {
|
|
174
|
+
try {
|
|
175
|
+
// save the page to ensure its id
|
|
176
|
+
const savedPage = await this.saveData(page, true);
|
|
177
|
+
// create chunks
|
|
178
|
+
const docChunk_DS = this.getDocChunkDataService();
|
|
179
|
+
const chunksCompareResults = await docChunk_DS.processNewPage(savedPage);
|
|
180
|
+
/* const chunks: T_Chunk[] = await docChunk_DS.processNewChunks(page); */
|
|
181
|
+
/* page.chunks.forEach((chunk: T_Chunk) => {
|
|
182
|
+
chunk.documentId = savedPage.documentId;
|
|
183
|
+
chunk.pageId = savedPage._id;
|
|
184
|
+
});
|
|
185
|
+
page._id = savedPage._id; */
|
|
186
|
+
const pageCompareResult = {
|
|
187
|
+
result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
|
|
188
|
+
newPage: page,
|
|
189
|
+
oldPage: null,
|
|
190
|
+
chunkCompareResults: chunksCompareResults,
|
|
191
|
+
reports: [
|
|
192
|
+
`New page: ${page.name}`,
|
|
193
|
+
`${page.chunks.length} new chunks in ${page.name}`,
|
|
194
|
+
],
|
|
195
|
+
};
|
|
196
|
+
return pageCompareResult;
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
200
|
+
...this.getDefaultErrorSettings('processNewPage', error),
|
|
201
|
+
errorCode: 'DCH-CSB-PP0',
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* this is a deep compare, that will update chunks if needed
|
|
207
|
+
* */
|
|
208
|
+
async comparePage(newPage, oldPage) {
|
|
209
|
+
try {
|
|
210
|
+
if (!oldPage) {
|
|
211
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
212
|
+
...this.getDefaultErrorSettings('comparePage', new Error('oldPage is required')),
|
|
213
|
+
errorCode: 'DCH-CSB-CP1',
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
if (this.debugLog) {
|
|
217
|
+
fsm_dynamo_1.DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${newPage.name}`);
|
|
218
|
+
}
|
|
219
|
+
let pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal;
|
|
220
|
+
const messages = [];
|
|
221
|
+
const chunkCompareResults = [];
|
|
222
|
+
// Compare all relevant fields for comprehensive change detection
|
|
223
|
+
if (newPage.name !== oldPage.name) {
|
|
224
|
+
pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
225
|
+
messages.push(`Page name modified`);
|
|
226
|
+
}
|
|
227
|
+
if (newPage.content !== oldPage.content) {
|
|
228
|
+
pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
229
|
+
messages.push(`Page content modified`);
|
|
230
|
+
}
|
|
231
|
+
// Compare allFlaggedParents arrays
|
|
232
|
+
const allFlaggedParentsEqual = this.compareStringArrays(newPage.allFlaggedParents || [], oldPage.allFlaggedParents || []);
|
|
233
|
+
if (!allFlaggedParentsEqual) {
|
|
234
|
+
pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
235
|
+
messages.push(`Page allFlaggedParents modified`);
|
|
236
|
+
}
|
|
237
|
+
// Compare allFlaggedParentsMerged
|
|
238
|
+
if (newPage.allFlaggedParentsMerged !== oldPage.allFlaggedParentsMerged) {
|
|
239
|
+
pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
240
|
+
messages.push(`Page allFlaggedParentsMerged modified`);
|
|
241
|
+
}
|
|
242
|
+
// Compare filePath
|
|
243
|
+
if (newPage.filePath !== oldPage.filePath) {
|
|
244
|
+
pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
245
|
+
messages.push(`Page filePath modified`);
|
|
246
|
+
}
|
|
247
|
+
// Compare parentedPath arrays
|
|
248
|
+
const parentedPathEqual = this.compareStringArrays(newPage.parentedPath || [], oldPage.parentedPath || []);
|
|
249
|
+
if (!parentedPathEqual) {
|
|
250
|
+
pageCompareResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
|
|
251
|
+
messages.push(`Page parentedPath modified`);
|
|
252
|
+
}
|
|
253
|
+
const result = {
|
|
254
|
+
result: pageCompareResult,
|
|
255
|
+
newPage: newPage,
|
|
256
|
+
oldPage: oldPage,
|
|
257
|
+
chunkCompareResults: chunkCompareResults,
|
|
258
|
+
reports: messages,
|
|
259
|
+
};
|
|
260
|
+
return result;
|
|
261
|
+
}
|
|
262
|
+
catch (error) {
|
|
263
|
+
throw new fsm_dynamo_1.DyFM_Error({
|
|
264
|
+
...this.getDefaultErrorSettings('comparePage', error),
|
|
265
|
+
errorCode: 'DCH-CSB-CP0',
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
compareStringArrays(arr1, arr2) {
|
|
270
|
+
if (arr1.length !== arr2.length) {
|
|
271
|
+
return false;
|
|
272
|
+
}
|
|
273
|
+
return arr1.every((item, index) => item === arr2[index]);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
exports.DyNTS_OAI_DocPageDataService = DyNTS_OAI_DocPageDataService;
|
|
277
|
+
//# sourceMappingURL=oai-doc-page.data-service.js.map
|
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oai-doc-page.data-service.js","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.ts"],"names":[],"mappings":";;;AAAA,sDAA8J;AAG9J,gFAAkF;AAKlF,8GAAmI;AAGnI,6EAA6E;AAC7E,2EAA4E;AAC5E,2GAAwG;AAGxG,MAAa,4BAKX,SAAQ,gCAAyB;IAOvB;IALV,QAAQ,GAAY,IAAI,CAAC;IACzB,wBAAwB,GAAY,IAAI,CAAC;IAEzC,YACE,MAAc,EACN,iCAAoD,EAC5D,IAAa,EACb,aAA4C,sDAA6D;QAEzG,KAAK,CACH,IAAI,EACJ,UAAU,EACV,MAAM,CACP,CAAC;QARM,sCAAiC,GAAjC,iCAAiC,CAAmB;IAS9D,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,yDAA6B,CACtC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iCAAiC,CACvC,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,0DAA6B,CACtC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iCAAiC,CACvC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAkB,EAAE,KAAe;QACpD,IAAI,CAAC;YACH,MAAM,cAAc,GAAsD,EAAE,CAAC;YAE7E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;gBAC7B,cAAc,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YACpD,CAAC;YAED,yFAAyF;YACzF,MAAM,YAAY,GAAa,MAAM,IAAI,CAAC,YAAY,CAAC;gBACrD,UAAU,EAAE,UAAU;gBACtB,GAAG,EAAE;oBACH,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;iBAClC;aACuB,CAAC,CAAC;YAE5B,MAAM,WAAW,GACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAEhC,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;gBAE9B,MAAM,cAAc,GAAc,MAAM,WAAW,CAAC,YAAY,CAAC;oBAC/D,UAAU,EAAE,UAAU;oBACtB,MAAM,EAAE,WAAW,CAAC,GAAG;iBACQ,CAAC,CAAC;gBAEnC,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;oBACnC,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACzE,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,iBAAiB,CAAC,CAAC;gBAEzD,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACtE,QAAQ,CAAC,OAAO,CAAC,SAAS,WAAW,CAAC,IAAI,sBAAsB,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;gBAE1F,cAAc,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,2DAA4B,CAAC,OAAO;oBAC5C,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,WAAW;oBACpB,mBAAmB,EAAE,EAAE;oBACvB,OAAO,EAAE,QAAQ;iBAClB,CAAC,CAAC;YACL,CAAC;YAED,OAAO,cAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,cAAc,EAAE,KAAK,CAAC;gBACtD,SAAS,EAAE,aAAa;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAY;QAC5B,IAAI,CAAC;YACH,IAAI,YAAoB,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACb,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC;oBACjC,GAAG,EAAE,IAAI,CAAC,GAAG;iBACiB,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;oBACzB,MAAM,IAAI,uBAAU,CAAC;wBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;wBACvF,SAAS,EAAE,aAAa;qBACzB,CAAC,CAAC;gBACL,CAAC;gBAED,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC;oBACjC,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;iBACL,CAAC,CAAC;YACpC,CAAC;YAED,IAAI,iBAAkE,CAAC;YAEvE,IAAI,YAAY,EAAE,CAAC;gBACjB,gDAAgD;gBAChD,iBAAiB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBAE/D,2EAA2E;gBAC3E,MAAM,WAAW,GACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAEhC,MAAM,oBAAoB,GAA+C,MAAM,WAAW,CAAC,WAAW,CACpG,YAAY,CACb,CAAC;gBAEF,iBAAiB,CAAC,mBAAmB,GAAG,oBAAoB,CAAC;gBAE7D,mEAAmE;gBACnE,IACE,iBAAiB,CAAC,MAAM,KAAK,2DAA4B,CAAC,KAAK;oBAC/D,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,2DAA4B,CAAC,KAAK,CAAC,EAC/E,CAAC;oBACD,iBAAiB,CAAC,MAAM,GAAG,2DAA4B,CAAC,QAAQ,CAAC;oBACjE,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACpD,CAAC;gBAED,IAAI,iBAAiB,CAAC,MAAM,KAAK,2DAA4B,CAAC,KAAK,EAAE,CAAC;oBACpE,0DAA0D;oBAC1D,iBAAiB,CAAC,OAAO,GAAG,YAAY,CAAC;oBACzC,iBAAiB,CAAC,OAAO,GAAG,YAAY,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,0CAA0C;oBAC1C,IAAI,cAAsB,CAAC;oBAC3B,cAAc,GAAG,YAAY,CAAC;oBAC9B,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;oBACpC,OAAO,cAAc,CAAC,MAAM,CAAC;oBAE7B,MAAM,SAAS,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;oBAEpE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAc,EAAE,EAAE;wBACrC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;wBACxC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;oBAC/B,CAAC,CAAC,CAAC;oBACH,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC;oBACzB,iBAAiB,CAAC,OAAO,GAAG,IAAI,CAAC;gBACnC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,oCAAoC;gBACpC,MAAM,cAAc,GAAW;oBAC7B,GAAG,IAAI;iBACR,CAAC;gBACF,OAAO,cAAc,CAAC,MAAM,CAAC;gBAE7B,MAAM,SAAS,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;gBAEpE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAc,EAAE,EAAE;oBACrC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;oBACxC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;gBAC/B,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC;gBAEzB,iBAAiB,GAAG;oBAClB,MAAM,EAAE,2DAA4B,CAAC,GAAG;oBACxC,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,EAAE;oBACvB,OAAO,EAAE;wBACP,aAAa,IAAI,CAAC,IAAI,EAAE;wBACxB,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,kBAAkB,IAAI,CAAC,IAAI,EAAE;qBACnD;iBACF,CAAC;gBAEF,MAAM,WAAW,GACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAEhC,MAAM,oBAAoB,GAA+C,MAAM,WAAW,CAAC,cAAc,CACvG,SAAS,CACV,CAAC;gBAEF,iBAAiB,CAAC,mBAAmB,GAAG,oBAAoB,CAAC;YAC/D,CAAC;YAED,OAAO,iBAAiB,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,KAAK,CAAC;gBACrD,SAAS,EAAE,aAAa;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAe;QACnC,IAAI,CAAC;YACH,MAAM,kBAAkB,GAAsD,EAAE,CAAC;YAEjF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,iBAAiB,GAAoD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC3G,kBAAkB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAC7C,CAAC;YAED,OAAO,kBAAkB,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,KAAK,CAAC;gBACzD,SAAS,EAAE,aAAa;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAY;QAC/B,IAAI,CAAC;YACH,iCAAiC;YACjC,MAAM,SAAS,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAE1D,gBAAgB;YAChB,MAAM,WAAW,GACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAEhC,MAAM,oBAAoB,GAA+C,MAAM,WAAW,CAAC,cAAc,CACvG,SAAS,CACV,CAAC;YACF,yEAAyE;YAEzE;;;;wCAI4B;YAE5B,MAAM,iBAAiB,GAAoD;gBACzE,MAAM,EAAE,2DAA4B,CAAC,GAAG;gBACxC,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,IAAI;gBACb,mBAAmB,EAAE,oBAAoB;gBACzC,OAAO,EAAE;oBACP,aAAa,IAAI,CAAC,IAAI,EAAE;oBACxB,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,kBAAkB,IAAI,CAAC,IAAI,EAAE;iBACnD;aACF,CAAC;YAEF,OAAO,iBAAiB,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,EAAE,KAAK,CAAC;gBACxD,SAAS,EAAE,aAAa;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;SAEK;IACL,KAAK,CAAC,WAAW,CAAC,OAAe,EAAE,OAAe;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,uBAAU,CAAC;oBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;oBAChF,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,qBAAQ,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,CAAC,QAAQ,UAAU,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,IAAI,iBAAiB,GAAiC,2DAA4B,CAAC,KAAK,CAAC;YACzF,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,MAAM,mBAAmB,GAA+C,EAAE,CAAC;YAE3E,iEAAiE;YACjE,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;gBAClC,iBAAiB,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;gBACxC,iBAAiB,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzC,CAAC;YAED,mCAAmC;YACnC,MAAM,sBAAsB,GAAG,IAAI,CAAC,mBAAmB,CACrD,OAAO,CAAC,iBAAiB,IAAI,EAAE,EAC/B,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAChC,CAAC;YACF,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC5B,iBAAiB,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YACnD,CAAC;YAED,kCAAkC;YAClC,IAAI,OAAO,CAAC,uBAAuB,KAAK,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBACxE,iBAAiB,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACzD,CAAC;YAED,mBAAmB;YACnB,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC1C,iBAAiB,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,8BAA8B;YAC9B,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAChD,OAAO,CAAC,YAAY,IAAI,EAAE,EAC1B,OAAO,CAAC,YAAY,IAAI,EAAE,CAC3B,CAAC;YACF,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACvB,iBAAiB,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC9C,CAAC;YAGD,MAAM,MAAM,GAAoD;gBAC9D,MAAM,EAAE,iBAAiB;gBACzB,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,OAAO;gBAChB,mBAAmB,EAAE,mBAAmB;gBACxC,OAAO,EAAE,QAAQ;aAClB,CAAC;YAGF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,KAAK,CAAC;gBACrD,SAAS,EAAE,aAAa;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,IAAc,EAAE,IAAc;QACxD,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3D,CAAC;CA+MF;AA1iBD,oEA0iBC"}
|
package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { DyFM_DataModel_Params } from '@futdevpro/fsm-dynamo';
|
|
2
|
+
import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
|
|
3
|
+
import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
|
|
4
|
+
import { DyNTS_DAI_DocPage } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
|
|
5
|
+
import { DyNTS_DAI_Document } from '../../../document-ai/_models/data-models/dai-document.data-model';
|
|
6
|
+
import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
|
|
7
|
+
import { DyNTS_OAI_DocPageDataService } from './oai-doc-page.data-service';
|
|
8
|
+
import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
|
|
9
|
+
import { DyNTS_DAI_DocumentCompareResult } from '../../../document-ai/_models/interfaces/dai-document-compare-result.interface';
|
|
10
|
+
export declare class DyNTS_OAI_DocumentDataService<T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk, T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, T_ExternalDocumentId = string, T_ExternalMetadata = any, T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata>> extends DyNTS_DataService<T_Document> {
|
|
11
|
+
private docChunkDataServiceOpenAiSettings;
|
|
12
|
+
debugLog: boolean;
|
|
13
|
+
absoluteDeletesOnProcess: boolean;
|
|
14
|
+
constructor(issuer: string, docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings, data?: T_Document, dataParams?: DyFM_DataModel_Params<T_Document>);
|
|
15
|
+
getDocPageDataService(): DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>;
|
|
16
|
+
getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>;
|
|
17
|
+
processDocuments(documents: T_Document[]): Promise<DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[]>;
|
|
18
|
+
processDocument(document: T_Document): Promise<DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>>;
|
|
19
|
+
compareDocument_deep(newDocument: T_Document, oldDocument: T_Document): Promise<DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>>;
|
|
20
|
+
compareDocument_flat(newDocument: T_Document, oldDocument: T_Document): Promise<DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>>;
|
|
21
|
+
reportDocumentProcessResult(documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=oai-document.data-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oai-document.data-service.d.ts","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,qBAAqB,EAAsF,MAAM,uBAAuB,CAAC;AAC1K,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,mEAAmE,CAAC;AACvG,OAAO,EAAE,iBAAiB,EAAE,MAAM,kEAAkE,CAAC;AACrG,OAAO,EAAE,kBAAkB,EAAiC,MAAM,kEAAkE,CAAC;AACrI,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,4BAA4B,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,+BAA+B,EAAE,MAAM,+EAA+E,CAAC;AAKhI,qBAAa,6BAA6B,CACxC,OAAO,SAAS,kBAAkB,GAAG,kBAAkB,EACvD,MAAM,SAAS,iBAAiB,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,EACtE,oBAAoB,GAAG,MAAM,EAC7B,kBAAkB,GAAG,GAAG,EACxB,UAAU,SAAS,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,CAAC,CAChL,SAAQ,iBAAiB,CAAC,UAAU,CAAC;IAOnC,OAAO,CAAC,iCAAiC;IAL3C,QAAQ,EAAE,OAAO,CAAQ;IACzB,wBAAwB,EAAE,OAAO,CAAQ;gBAGvC,MAAM,EAAE,MAAM,EACN,iCAAiC,EAAE,iBAAiB,EAC5D,IAAI,CAAC,EAAE,UAAU,EACjB,UAAU,GAAE,qBAAqB,CAAC,UAAU,CAAsE;IASpH,qBAAqB,IAAI,4BAA4B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,UAAU,CAAC;IAOxG,sBAAsB,IAAI,6BAA6B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,UAAU,CAAC;IAOpG,gBAAgB,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,OAAO,CACtD,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,UAAU,CAAC,EAAE,CACzG;IAyEK,eAAe,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAClD,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,UAAU,CAAC,CACvG;IA+FK,oBAAoB,CAAC,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,GAAG,OAAO,CACnF,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,UAAU,CAAC,CACvG;IA2GK,oBAAoB,CAAC,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,GAAG,OAAO,CACnF,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,UAAU,CAAC,CACvG;IAyDK,2BAA2B,CAC/B,qBAAqB,EAAE,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,UAAU,CAAC,GAC5H,OAAO,CAAC,IAAI,CAAC;CAqCjB"}
|