@futdevpro/nts-dynamo 1.15.154 → 1.15.156

package/build/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.js

@@ -1,198 +1,198 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DyNTS_LVS_DocChunkDataService = void 0;
-const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
-const lvs_local_vector_search_data_service_1 = require("./lvs-local-vector-search.data-service");
-const dai_doc_chunk_data_model_1 = require("../../ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model");
-const oai_document_data_service_1 = require("../../ai/_modules/open-ai/_services/data-services/oai-document.data-service");
-const oai_doc_page_data_service_1 = require("../../ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service");
-const dai_compare_result_type_enum_1 = require("../../ai/_modules/document-ai/_enums/dai-compare-result-type.enum");
-const dai_document_util_1 = require("../../ai/_modules/document-ai/_collections/dai-document.util");
-const dai_chunking_util_1 = require("../../ai/_modules/document-ai/_collections/dai-chunking.util");
-/**
- * Local Doc Chunk Data Service
- * Extends {@link DyNTS_LVS_VectorDataService} to provide document chunking functionality
- * with local in-memory vector search instead of MongoDB vector search
- */
-class DyNTS_LVS_DocChunkDataService extends lvs_local_vector_search_data_service_1.DyNTS_LVS_VectorDataService {
-    /* override debugLog: boolean = false; */
-    clickupWorkspaceId;
-    constructor(issuer, openAiSettings, dataParams = dai_doc_chunk_data_model_1.DyNTS_DAI_docChunk_dataParams) {
-        super(null, dataParams, openAiSettings, issuer);
-    }
-    getDocumentDataService() {
-        return new oai_document_data_service_1.DyNTS_OAI_DocumentDataService(this.issuer, this.openAISettings);
-    }
-    getDocPageDataService() {
-        return new oai_doc_page_data_service_1.DyNTS_OAI_DocPageDataService(this.issuer, this.openAISettings);
-    }
-    async processPage(page) {
-        try {
-            const newChunks = await this.chunkPage(page);
-            const oldChunks = await this.findDataList({
-                /* pageId: page._id, */
-                externalPageId: page.externalPageId,
-            });
-            oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
-            const chunkCompareResults = [];
-            for (let i = 0; i < newChunks.length; i++) {
-                const newChunk = newChunks[i];
-                newChunk.documentId = page.documentId;
-                newChunk.pageId = page._id;
-                newChunk.chunkIndex = i;
-                const oldChunk = oldChunks[i];
-                if (oldChunk) {
-                    newChunk._id = oldChunk._id;
-                    // Compare first to determine if we need to save
-                    const chunkCompareResult = await this.compareChunk(newChunk, oldChunk);
-                    if (chunkCompareResult.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
-                        // No changes detected, reuse existing chunk without saving
-                        chunkCompareResult.newChunk = oldChunk;
-                        chunkCompareResults.push(chunkCompareResult);
-                    }
-                    else {
-                        // Changes detected, save the updated chunk
-                        const savedChunk = await this.saveData(newChunk, true);
-                        chunkCompareResult.newChunk = savedChunk;
-                        chunkCompareResults.push(chunkCompareResult);
-                    }
-                }
-                else {
-                    // New chunk, save it
-                    const savedChunk = await this.saveData(newChunk, true);
-                    chunkCompareResults.push({
-                        result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
-                        newChunk: savedChunk,
-                        oldChunk: null,
-                    });
-                }
-            }
-            for (let i = newChunks.length; i < oldChunks.length; i++) {
-                const oldChunk = oldChunks[i];
-                await this.deleteData(oldChunk._id);
-                chunkCompareResults.push({
-                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted,
-                    newChunk: null,
-                    oldChunk: oldChunk,
-                });
-            }
-            return chunkCompareResults;
-        }
-        catch (error) {
-            throw new fsm_dynamo_1.DyFM_Error({
-                ...this.getDefaultErrorSettings('processPage', error),
-                errorCode: 'DCH-CSB-PP0',
-            });
-        }
-    }
-    async processNewPage(page) {
-        try {
-            const chunks = await this.chunkPage(page);
-            const chunkCompareResults = [];
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                chunk.documentId = page.documentId;
-                chunk.pageId = page._id;
-                chunk.chunkIndex = i;
-                const savedChunk = await this.saveData(chunk, true);
-                chunkCompareResults.push({
-                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
-                    newChunk: savedChunk,
-                    oldChunk: null,
-                });
-            }
-            return chunkCompareResults;
-        }
-        catch (error) {
-            throw new fsm_dynamo_1.DyFM_Error({
-                ...this.getDefaultErrorSettings('processNewPage', error),
-                errorCode: 'DCH-CSB-PNP0',
-            });
-        }
-    }
-    async compareChunk(newChunk, oldChunk) {
-        try {
-            if (!oldChunk) {
-                throw new fsm_dynamo_1.DyFM_Error({
-                    ...this.getDefaultErrorSettings('compareChunk', new Error('oldChunk is required')),
-                    errorCode: 'DCH-CSB-CC1',
-                });
-            }
-            if (this.debugLog) {
-                fsm_dynamo_1.DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
-            }
-            // Compare all relevant fields for comprehensive change detection
-            const contentEqual = oldChunk.chunkContent === newChunk.chunkContent;
-            const pathEqual = oldChunk.path === newChunk.path;
-            const documentNameEqual = oldChunk.documentName === newChunk.documentName;
-            const pageNameEqual = oldChunk.pageName === newChunk.pageName;
-            const pageLinkEqual = oldChunk.pageLink === newChunk.pageLink;
-            // Compare allFlaggedParents arrays
-            const allFlaggedParentsEqual = this.compareStringArrays(oldChunk.allFlaggedParents || [], newChunk.allFlaggedParents || []);
-            if (contentEqual &&
-                pathEqual &&
-                documentNameEqual &&
-                pageNameEqual &&
-                pageLinkEqual &&
-                allFlaggedParentsEqual) {
-                return {
-                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal,
-                    newChunk: newChunk,
-                    oldChunk: oldChunk,
-                };
-            }
-            return {
-                result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified,
-                newChunk: newChunk,
-                oldChunk: oldChunk,
-            };
-        }
-        catch (error) {
-            throw new fsm_dynamo_1.DyFM_Error({
-                ...this.getDefaultErrorSettings('compareChunk', error),
-                errorCode: 'DCH-CSB-CC0',
-            });
-        }
-    }
-    compareStringArrays(arr1, arr2) {
-        if (arr1.length !== arr2.length) {
-            return false;
-        }
-        return arr1.every((item, index) => item === arr2[index]);
-    }
-    async chunkPage(page) {
-        try {
-            page.chunks = await dai_chunking_util_1.DyNTS_DAI_DocChunking_Util.chunkMdContent(page, (page) => this.getPageLink(page), this.issuer);
-            if (this.debugLog) {
-                fsm_dynamo_1.DyFM_Log.H_info(`Processing file: ${page.name}`, {
-                    'Page name': page.name,
-                    'Page filePath': page.filePath,
-                    'Page parentedPath': page.parentedPath,
-                    /* 'Page content': page.content,  */
-                    'Page parentFlags': '\n' + page.allFlaggedParents.join('\n') + '\n',
-                    chunks: page.chunks.map(chunk => {
-                        return {
-                            'Chunk name': chunk.documentName,
-                            'Chunk path': chunk.path,
-                            'Chunk index': chunk.chunkIndex,
-                            /* 'Chunk content': chunk.chunkContent, */
-                            'Chunk parentFlags': '\n' + chunk.allFlaggedParents.join('\n') + '\n',
-                        };
-                    })
-                });
-            }
-            return page.chunks;
-        }
-        catch (error) {
-            throw new fsm_dynamo_1.DyFM_Error({
-                ...this.getDefaultErrorSettings('chunkDocPages', error),
-                errorCode: 'DCH-CSB-CP0',
-            });
-        }
-    }
-    getPageLink(page) {
-        return dai_document_util_1.DyNTS_DAI_Document_Util.getClickupPageLink(page.name, this.clickupWorkspaceId, page.externalDocumentId, page.externalPageId);
-    }
-}
-exports.DyNTS_LVS_DocChunkDataService = DyNTS_LVS_DocChunkDataService;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DyNTS_LVS_DocChunkDataService = void 0;
+const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
+const lvs_local_vector_search_data_service_1 = require("./lvs-local-vector-search.data-service");
+const dai_doc_chunk_data_model_1 = require("../../ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model");
+const oai_document_data_service_1 = require("../../ai/_modules/open-ai/_services/data-services/oai-document.data-service");
+const oai_doc_page_data_service_1 = require("../../ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service");
+const dai_compare_result_type_enum_1 = require("../../ai/_modules/document-ai/_enums/dai-compare-result-type.enum");
+const dai_document_util_1 = require("../../ai/_modules/document-ai/_collections/dai-document.util");
+const dai_chunking_util_1 = require("../../ai/_modules/document-ai/_collections/dai-chunking.util");
+/**
+ * Local Doc Chunk Data Service
+ * Extends {@link DyNTS_LVS_VectorDataService} to provide document chunking functionality
+ * with local in-memory vector search instead of MongoDB vector search
+ */
+class DyNTS_LVS_DocChunkDataService extends lvs_local_vector_search_data_service_1.DyNTS_LVS_VectorDataService {
+    /* override debugLog: boolean = false; */
+    clickupWorkspaceId;
+    constructor(issuer, openAiSettings, dataParams = dai_doc_chunk_data_model_1.DyNTS_DAI_docChunk_dataParams) {
+        super(null, dataParams, openAiSettings, issuer);
+    }
+    getDocumentDataService() {
+        return new oai_document_data_service_1.DyNTS_OAI_DocumentDataService(this.issuer, this.openAISettings);
+    }
+    getDocPageDataService() {
+        return new oai_doc_page_data_service_1.DyNTS_OAI_DocPageDataService(this.issuer, this.openAISettings);
+    }
+    async processPage(page) {
+        try {
+            const newChunks = await this.chunkPage(page);
+            const oldChunks = await this.findDataList({
+                /* pageId: page._id, */
+                externalPageId: page.externalPageId,
+            });
+            oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
+            const chunkCompareResults = [];
+            for (let i = 0; i < newChunks.length; i++) {
+                const newChunk = newChunks[i];
+                newChunk.documentId = page.documentId;
+                newChunk.pageId = page._id;
+                newChunk.chunkIndex = i;
+                const oldChunk = oldChunks[i];
+                if (oldChunk) {
+                    newChunk._id = oldChunk._id;
+                    // Compare first to determine if we need to save
+                    const chunkCompareResult = await this.compareChunk(newChunk, oldChunk);
+                    if (chunkCompareResult.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
+                        // No changes detected, reuse existing chunk without saving
+                        chunkCompareResult.newChunk = oldChunk;
+                        chunkCompareResults.push(chunkCompareResult);
+                    }
+                    else {
+                        // Changes detected, save the updated chunk
+                        const savedChunk = await this.saveData(newChunk, true);
+                        chunkCompareResult.newChunk = savedChunk;
+                        chunkCompareResults.push(chunkCompareResult);
+                    }
+                }
+                else {
+                    // New chunk, save it
+                    const savedChunk = await this.saveData(newChunk, true);
+                    chunkCompareResults.push({
+                        result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
+                        newChunk: savedChunk,
+                        oldChunk: null,
+                    });
+                }
+            }
+            for (let i = newChunks.length; i < oldChunks.length; i++) {
+                const oldChunk = oldChunks[i];
+                await this.deleteData(oldChunk._id);
+                chunkCompareResults.push({
+                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted,
+                    newChunk: null,
+                    oldChunk: oldChunk,
+                });
+            }
+            return chunkCompareResults;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('processPage', error),
+                errorCode: 'DCH-CSB-PP0',
+            });
+        }
+    }
+    async processNewPage(page) {
+        try {
+            const chunks = await this.chunkPage(page);
+            const chunkCompareResults = [];
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                chunk.documentId = page.documentId;
+                chunk.pageId = page._id;
+                chunk.chunkIndex = i;
+                const savedChunk = await this.saveData(chunk, true);
+                chunkCompareResults.push({
+                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
+                    newChunk: savedChunk,
+                    oldChunk: null,
+                });
+            }
+            return chunkCompareResults;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('processNewPage', error),
+                errorCode: 'DCH-CSB-PNP0',
+            });
+        }
+    }
+    async compareChunk(newChunk, oldChunk) {
+        try {
+            if (!oldChunk) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('compareChunk', new Error('oldChunk is required')),
+                    errorCode: 'DCH-CSB-CC1',
+                });
+            }
+            if (this.debugLog) {
+                fsm_dynamo_1.DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
+            }
+            // Compare all relevant fields for comprehensive change detection
+            const contentEqual = oldChunk.chunkContent === newChunk.chunkContent;
+            const pathEqual = oldChunk.path === newChunk.path;
+            const documentNameEqual = oldChunk.documentName === newChunk.documentName;
+            const pageNameEqual = oldChunk.pageName === newChunk.pageName;
+            const pageLinkEqual = oldChunk.pageLink === newChunk.pageLink;
+            // Compare allFlaggedParents arrays
+            const allFlaggedParentsEqual = this.compareStringArrays(oldChunk.allFlaggedParents || [], newChunk.allFlaggedParents || []);
+            if (contentEqual &&
+                pathEqual &&
+                documentNameEqual &&
+                pageNameEqual &&
+                pageLinkEqual &&
+                allFlaggedParentsEqual) {
+                return {
+                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal,
+                    newChunk: newChunk,
+                    oldChunk: oldChunk,
+                };
+            }
+            return {
+                result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified,
+                newChunk: newChunk,
+                oldChunk: oldChunk,
+            };
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('compareChunk', error),
+                errorCode: 'DCH-CSB-CC0',
+            });
+        }
+    }
+    compareStringArrays(arr1, arr2) {
+        if (arr1.length !== arr2.length) {
+            return false;
+        }
+        return arr1.every((item, index) => item === arr2[index]);
+    }
+    async chunkPage(page) {
+        try {
+            page.chunks = await dai_chunking_util_1.DyNTS_DAI_DocChunking_Util.chunkMdContent(page, (page) => this.getPageLink(page), this.issuer);
+            if (this.debugLog) {
+                fsm_dynamo_1.DyFM_Log.H_info(`Processing file: ${page.name}`, {
+                    'Page name': page.name,
+                    'Page filePath': page.filePath,
+                    'Page parentedPath': page.parentedPath,
+                    /* 'Page content': page.content,  */
+                    'Page parentFlags': '\n' + page.allFlaggedParents.join('\n') + '\n',
+                    chunks: page.chunks.map(chunk => {
+                        return {
+                            'Chunk name': chunk.documentName,
+                            'Chunk path': chunk.path,
+                            'Chunk index': chunk.chunkIndex,
+                            /* 'Chunk content': chunk.chunkContent, */
+                            'Chunk parentFlags': '\n' + chunk.allFlaggedParents.join('\n') + '\n',
+                        };
+                    })
+                });
+            }
+            return page.chunks;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('chunkDocPages', error),
+                errorCode: 'DCH-CSB-CP0',
+            });
+        }
+    }
+    getPageLink(page) {
+        return dai_document_util_1.DyNTS_DAI_Document_Util.getClickupPageLink(page.name, this.clickupWorkspaceId, page.externalDocumentId, page.externalPageId);
+    }
+}
+exports.DyNTS_LVS_DocChunkDataService = DyNTS_LVS_DocChunkDataService;
 //# sourceMappingURL=lvs-doc-chunk-data.service.js.map