@futdevpro/nts-dynamo 1.14.28 → 1.14.29

package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.js

@@ -0,0 +1,319 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DyNTS_OAI_DocumentDataService = void 0;
+const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
+const data_service_1 = require("../../../../../../_services/base/data.service");
+const dai_document_data_model_1 = require("../../../document-ai/_models/data-models/dai-document.data-model");
+const oai_doc_chunk_data_service_1 = require("./oai-doc-chunk-data.service");
+const oai_doc_page_data_service_1 = require("./oai-doc-page.data-service");
+const dai_compare_result_type_enum_1 = require("../../../document-ai/_enums/dai-compare-result-type.enum");
+class DyNTS_OAI_DocumentDataService extends data_service_1.DyNTS_DataService {
+    docChunkDataServiceOpenAiSettings;
+    debugLog = true;
+    absoluteDeletesOnProcess = true;
+    constructor(issuer, docChunkDataServiceOpenAiSettings, data, dataParams = dai_document_data_model_1.DyNTS_DAI_document_dataParams) {
+        super(data, dataParams, issuer);
+        this.docChunkDataServiceOpenAiSettings = docChunkDataServiceOpenAiSettings;
+    }
+    getDocPageDataService() {
+        return new oai_doc_page_data_service_1.DyNTS_OAI_DocPageDataService(this.issuer, this.docChunkDataServiceOpenAiSettings);
+    }
+    getDocChunkDataService() {
+        return new oai_doc_chunk_data_service_1.DyNTS_OAI_DocChunkDataService(this.issuer, this.docChunkDataServiceOpenAiSettings);
+    }
+    async processDocuments(documents) {
+        try {
+            if (!documents || documents.length === 0) {
+                return [];
+            }
+            const compareResults = [];
+            await fsm_dynamo_1.DyFM_Array.asyncForEachAllAtOnce(documents, async (document) => {
+                compareResults.push(await this.processDocument(document));
+            });
+            const deletedDocuments = await this.findDataList({
+                _id: {
+                    $nin: documents.map(document => document._id),
+                },
+            });
+            const docPage_DS = this.getDocPageDataService();
+            const docChunk_DS = this.getDocChunkDataService();
+            for (const deletedDocument of deletedDocuments) {
+                const messages = [];
+                const chunksToDelete = await docChunk_DS.findDataList({
+                    documentId: deletedDocument._id,
+                });
+                for (const chunk of chunksToDelete) {
+                    await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
+                }
+                messages.push(`${chunksToDelete.length} chunks deleted`);
+                const pagesToDelete = await docPage_DS.findDataList({
+                    documentId: deletedDocument._id,
+                });
+                for (const page of pagesToDelete) {
+                    await docPage_DS.deleteData(page._id, this.absoluteDeletesOnProcess);
+                }
+                messages.unshift(`${pagesToDelete.length} pages deleted`);
+                await this.deleteData(deletedDocument._id, this.absoluteDeletesOnProcess);
+                messages.unshift(`Document "${deletedDocument?.name}" deleted`);
+                compareResults.push({
+                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted,
+                    newDocument: null,
+                    oldDocument: deletedDocument,
+                    pageCompareResults: [],
+                    reports: messages,
+                });
+            }
+            for (const result of compareResults) {
+                // report the result
+                await this.reportDocumentProcessResult(result);
+            }
+            return compareResults;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('processDocuments', error),
+            });
+        }
+    }
+    async processDocument(document) {
+        try {
+            let existingDocument;
+            if (document._id) {
+                existingDocument = await this.findData({
+                    _id: document._id,
+                });
+            }
+            else {
+                if (!document.externalDocumentId) {
+                    throw new fsm_dynamo_1.DyFM_Error({
+                        ...this.getDefaultErrorSettings('processDocument', new Error('externalDocumentId is required')),
+                        errorCode: 'OAI-DSB-PD0',
+                    });
+                }
+                existingDocument = await this.findData({
+                    externalDocumentId: document.externalDocumentId,
+                });
+            }
+            let documentCompareResult;
+            if (existingDocument) {
+                // Set the existing document's ID to the new document and its pages before comparison
+                document._id = existingDocument._id;
+                document.pages.forEach((page) => {
+                    page.documentId = existingDocument._id;
+                });
+                // Compare first to determine if we need to save
+                documentCompareResult = await this.compareDocument_deep(document, existingDocument);
+                if (documentCompareResult.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
+                    // No changes detected, reuse existing document without saving
+                    documentCompareResult.newDocument = existingDocument;
+                    documentCompareResult.oldDocument = existingDocument;
+                }
+                else {
+                    // Changes detected, save the updated document
+                    let docDataToSave;
+                    docDataToSave = existingDocument;
+                    Object.assign(docDataToSave, document);
+                    delete docDataToSave.pages;
+                    const savedDocument = await this.saveData(docDataToSave, true);
+                    // set the id to the pages and the document
+                    document.pages.forEach((page) => {
+                        page.documentId = savedDocument._id;
+                    });
+                    document._id = savedDocument._id;
+                    documentCompareResult.newDocument = document;
+                }
+            }
+            else {
+                // New document, save it first to get ID
+                const docDataToSave = {
+                    ...document,
+                };
+                delete docDataToSave.pages;
+                const savedDocument = await this.saveData(docDataToSave, true);
+                // set the id to the pages and the document
+                document.pages.forEach((page) => {
+                    page.documentId = savedDocument._id;
+                });
+                document._id = savedDocument._id;
+                const docPage_DS = this.getDocPageDataService();
+                const pagesCompareResults = await docPage_DS.processNewPages(document.pages);
+                documentCompareResult = {
+                    result: dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new,
+                    newDocument: document,
+                    oldDocument: null,
+                    pageCompareResults: pagesCompareResults,
+                    reports: [
+                        `New document: "${document?.name}"`,
+                        `${document?.pages?.length ?? 0} new pages in "${document?.name}"`,
+                    ],
+                };
+            }
+            return documentCompareResult;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('processDocument', error),
+            });
+        }
+    }
+    async compareDocument_deep(newDocument, oldDocument) {
+        try {
+            if (this.debugLog) {
+                fsm_dynamo_1.DyFM_Log.info(`🔍 Comparing "${this.dataParams.dataName}" document: "${newDocument?.name}"`);
+            }
+            if (!oldDocument) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('compareDocument_deep', new Error('oldDocument is required')),
+                    errorCode: 'OAI-DSB-CD1',
+                });
+            }
+            const documentCompareResult_flat = await this.compareDocument_flat(newDocument, oldDocument);
+            const messages = documentCompareResult_flat.reports;
+            const docPage_DS = this.getDocPageDataService();
+            const pagesCompareResults = await docPage_DS.processPages(newDocument._id, newDocument.pages);
+            // determine the result based on the pages
+            let pagesResult;
+            if (pagesCompareResults.every(result => result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal)) {
+                pagesResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal;
+                messages.push(`All pages are equal`);
+            }
+            else if (pagesCompareResults.some(result => result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new ||
+                result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified ||
+                result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted)) {
+                pagesResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
+                const modifiedPages = pagesCompareResults.filter(result => result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified).map(result => result.newPage);
+                const deletedPages = pagesCompareResults.filter(result => result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted).map(result => result.oldPage);
+                const newPages = pagesCompareResults.filter(result => result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new).map(result => result.newPage);
+                /* const modifiedPagesCount: number = pagesCompareResults.filter(
+                  result => result.result === DyNTS_DAI_CompareResult_Type.modified
+                ).length;
+                const deletedPagesCount: number = pagesCompareResults.filter(
+                  result => result.result === DyNTS_DAI_CompareResult_Type.deleted
+                ).length;
+                const newPagesCount: number = pagesCompareResults.filter(
+                  result => result.result === DyNTS_DAI_CompareResult_Type.new
+                ).length; */
+                if (modifiedPages.length > 0) {
+                    messages.push(`${modifiedPages.length} modified pages: ${modifiedPages.map(page => `"${page.name}"`).join(', ')}`);
+                }
+                if (newPages.length > 0) {
+                    messages.push(`${newPages.length} new pages: ${newPages.map(page => `"${page.name}"`).join(', ')}`);
+                }
+                if (deletedPages.length > 0) {
+                    messages.push(`${deletedPages.length} deleted pages: ${deletedPages.map(page => `"${page.name}"`).join(', ')}`);
+                }
+            }
+            else {
+                pagesResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new;
+                const newPagesCount = pagesCompareResults.filter(result => result.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new).length;
+                messages.push(`${newPagesCount} new pages`);
+            }
+            // determine the result based on both the document and the pages
+            let documentResult;
+            if (documentCompareResult_flat.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new) {
+                documentResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new;
+            }
+            else if (documentCompareResult_flat.result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified) {
+                documentResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
+            }
+            else {
+                if (pagesResult === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
+                    documentResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal;
+                }
+                {
+                    documentResult = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
+                }
+            }
+            const result = {
+                ...documentCompareResult_flat,
+                result: documentResult,
+                newDocument: newDocument,
+                oldDocument: oldDocument,
+                pageCompareResults: pagesCompareResults,
+            };
+            return result;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('compareDocument', error),
+            });
+        }
+    }
+    async compareDocument_flat(newDocument, oldDocument) {
+        try {
+            if (!oldDocument) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('compareDocument_flat', new Error('oldDocument is required')),
+                    errorCode: 'OAI-DSB-CF1',
+                });
+            }
+            let result = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal;
+            const messages = [];
+            // Compare document name
+            if (newDocument.name !== oldDocument.name) {
+                result = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
+                messages.push(`Document name changed from "${oldDocument.name}" to "${newDocument.name}"`);
+            }
+            // Compare externalDocumentId
+            if (newDocument.externalDocumentId !== oldDocument.externalDocumentId) {
+                result = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
+                messages.push(`Document externalDocumentId changed`);
+            }
+            // Compare externalMetadata if present
+            if (newDocument.externalMetadata || oldDocument.externalMetadata) {
+                const newMetadataStr = newDocument.externalMetadata ? JSON.stringify(newDocument.externalMetadata) : '';
+                const oldMetadataStr = oldDocument.externalMetadata ? JSON.stringify(oldDocument.externalMetadata) : '';
+                if (newMetadataStr !== oldMetadataStr) {
+                    result = dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified;
+                    messages.push(`Document externalMetadata changed`);
+                }
+            }
+            if (result === dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal) {
+                messages.push(`Document name or not changed: "${newDocument.name}"`);
+            }
+            const result_flat = {
+                result: result,
+                newDocument: newDocument,
+                oldDocument: oldDocument,
+                pageCompareResults: [],
+                reports: messages,
+            };
+            return result_flat;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('compareDocument_shallow', error),
+            });
+        }
+    }
+    async reportDocumentProcessResult(documentCompareResult) {
+        try {
+            fsm_dynamo_1.DyFM_Log.log(`\n✅ Document processed: "${documentCompareResult?.newDocument?.name ?? documentCompareResult?.oldDocument?.name ?? 'unknown'}"`);
+            switch (documentCompareResult.result) {
+                case dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.new:
+                    fsm_dynamo_1.DyFM_Log.info(`  ✨ New document`);
+                    break;
+                case dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.equal:
+                    fsm_dynamo_1.DyFM_Log.success(`  🟰 Equal document`);
+                    break;
+                case dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.modified:
+                    fsm_dynamo_1.DyFM_Log.info(`  🔄 Modified document`);
+                    break;
+                case dai_compare_result_type_enum_1.DyNTS_DAI_CompareResult_Type.deleted:
+                    fsm_dynamo_1.DyFM_Log.warn(`  ❌ Deleted document`);
+                    break;
+            }
+            for (const report of documentCompareResult.reports) {
+                fsm_dynamo_1.DyFM_Log.log(`  - ${report}`);
+            }
+            fsm_dynamo_1.DyFM_Log.log('');
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('reportDocumentProcessing', error),
+            });
+        }
+    }
+}
+exports.DyNTS_OAI_DocumentDataService = DyNTS_OAI_DocumentDataService;
+//# sourceMappingURL=oai-document.data-service.js.map

package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"oai-document.data-service.js","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.ts"],"names":[],"mappings":";;;AAAA,sDAA0K;AAC1K,gFAAkF;AAGlF,8GAAqI;AACrI,6EAA6E;AAC7E,2EAA2E;AAI3E,2GAAwG;AAGxG,MAAa,6BAMX,SAAQ,gCAA6B;IAO3B;IALV,QAAQ,GAAY,IAAI,CAAC;IACzB,wBAAwB,GAAY,IAAI,CAAC;IAEzC,YACE,MAAc,EACN,iCAAoD,EAC5D,IAAiB,EACjB,aAAgD,uDAAkE;QAElH,KAAK,CACH,IAAI,EACJ,UAAU,EACV,MAAM,CACP,CAAC;QARM,sCAAiC,GAAjC,iCAAiC,CAAmB;IAS9D,CAAC;IAED,qBAAqB;QACnB,OAAO,IAAI,wDAA4B,CACrC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iCAAiC,CACvC,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,0DAA6B,CACtC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iCAAiC,CACvC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,SAAuB;QAG5C,IAAI,CAAC;YACH,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzC,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,cAAc,GAA6G,EAAE,CAAC;YAEpI,MAAM,uBAAU,CAAC,qBAAqB,CACpC,SAAS,EACT,KAAK,EAAE,QAAoB,EAAE,EAAE;gBAC7B,cAAc,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5D,CAAC,CACF,CAAC;YAEF,MAAM,gBAAgB,GAAiB,MAAM,IAAI,CAAC,YAAY,CAAC;gBAC7D,GAAG,EAAE;oBACH,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;iBAC9C;aACkC,CAAC,CAAC;YAEvC,MAAM,UAAU,GACd,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAE/B,MAAM,WAAW,GACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAEhC,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;gBAC/C,MAAM,QAAQ,GAAa,EAAE,CAAC;gBAE9B,MAAM,cAAc,GAAc,MAAM,WAAW,CAAC,YAAY,CAAC;oBAC/D,UAAU,EAAE,eAAe,CAAC,GAAG;iBACN,CAAC,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;oBACnC,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACzE,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,iBAAiB,CAAC,CAAC;gBAEzD,MAAM,aAAa,GAAa,MAAM,UAAU,CAAC,YAAY,CAAC;oBAC5D,UAAU,EAAE,eAAe,CAAC,GAAG;iBACP,CAAC,CAAC;gBAE5B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,MAAM,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACvE,CAAC;gBACD,QAAQ,CAAC,OAAO,CAAC,GAAG,aAAa,CAAC,MAAM,gBAAgB,CAAC,CAAC;gBAE1D,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBAC1E,QAAQ,CAAC,OAAO,CAAC,aAAa,eAAe,EAAE,IAAI,WAAW,CAAC,CAAC;gBAEhE,cAAc,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,2DAA4B,CAAC,OAAO;oBAC5C,WAAW,EAAE,IAAI;oBACjB,WAAW,EAAE,eAAe;oBAC5B,kBAAkB,EAAE,EAAE;oBACtB,OAAO,EAAE,QAAQ;iBAClB,CAAC,CAAC;YACL,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,oBAAoB;gBACpB,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC;YACjD,CAAC;YAED,OAAO,cAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,EAAE,KAAK,CAAC;aAC3D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,QAAoB;QAGxC,IAAI,CAAC;YACH,IAAI,gBAA4B,CAAC;YAEjC,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;gBACjB,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC;oBACrC,GAAG,EAAE,QAAQ,CAAC,GAAG;iBACiB,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;oBACjC,MAAM,IAAI,uBAAU,CAAC;wBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;wBAC/F,SAAS,EAAE,aAAa;qBACzB,CAAC,CAAC;gBACL,CAAC;gBAED,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC;oBACrC,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;iBAChD,CAAC,CAAC;YACL,CAAC;YAED,IAAI,qBAA6H,CAAC;YAElI,IAAI,gBAAgB,EAAE,CAAC;gBACrB,qFAAqF;gBACrF,QAAQ,CAAC,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC;gBACpC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,EAAE;oBACtC,IAAI,CAAC,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC;gBACzC,CAAC,CAAC,CAAC;gBAEH,gDAAgD;gBAChD,qBAAqB,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;gBAEpF,IAAI,qBAAqB,CAAC,MAAM,KAAK,2DAA4B,CAAC,KAAK,EAAE,CAAC;oBACxE,8DAA8D;oBAC9D,qBAAqB,CAAC,WAAW,GAAG,gBAAgB,CAAC;oBACrD,qBAAqB,CAAC,WAAW,GAAG,gBAAgB,CAAC;gBACvD,CAAC;qBAAM,CAAC;oBACN,8CAA8C;oBAC9C,IAAI,aAAyB,CAAC;oBAC9B,aAAa,GAAG,gBAAgB,CAAC;oBACjC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;oBACvC,OAAO,aAAa,CAAC,KAAK,CAAC;oBAE3B,MAAM,aAAa,GAAe,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;oBAE3E,2CAA2C;oBAC3C,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,EAAE;wBACtC,IAAI,CAAC,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC;oBACtC,CAAC,CAAC,CAAC;oBACH,QAAQ,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC;oBACjC,qBAAqB,CAAC,WAAW,GAAG,QAAQ,CAAC;gBAC/C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,wCAAwC;gBACxC,MAAM,aAAa,GAAe;oBAChC,GAAG,QAAQ;iBACZ,CAAC;gBACF,OAAO,aAAa,CAAC,KAAK,CAAC;gBAE3B,MAAM,aAAa,GAAe,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;gBAE3E,2CAA2C;gBAC3C,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,EAAE;oBACtC,IAAI,CAAC,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC;gBACtC,CAAC,CAAC,CAAC;gBACH,QAAQ,CAAC,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC;gBAEjC,MAAM,UAAU,GACd,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAE/B,MAAM,mBAAmB,GAAsD,MAAM,UAAU,CAAC,eAAe,CAC7G,QAAQ,CAAC,KAAK,CACf,CAAC;gBAEF,qBAAqB,GAAG;oBACtB,MAAM,EAAE,2DAA4B,CAAC,GAAG;oBACxC,WAAW,EAAE,QAAQ;oBACrB,WAAW,EAAE,IAAI;oBACjB,kBAAkB,EAAE,mBAAmB;oBACvC,OAAO,EAAE;wBACP,kBAAkB,QAAQ,EAAE,IAAI,GAAG;wBACnC,GAAG,QAAQ,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC,kBAAkB,QAAQ,EAAE,IAAI,GAAG;qBACnE;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,qBAAqB,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,KAAK,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,WAAuB,EAAE,WAAuB;QAGzE,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,qBAAQ,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,UAAU,CAAC,QAAQ,gBAAgB,WAAW,EAAE,IAAI,GAAG,CAAC,CAAC;YAC/F,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,uBAAU,CAAC;oBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,sBAAsB,EAAE,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;oBAC7F,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,0BAA0B,GAC9B,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YAC5D,MAAM,QAAQ,GAAa,0BAA0B,CAAC,OAAO,CAAC;YAE9D,MAAM,UAAU,GACd,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAE/B,MAAM,mBAAmB,GAAsD,MAAM,UAAU,CAAC,YAAY,CAC1G,WAAW,CAAC,GAAG,EACf,WAAW,CAAC,KAAK,CAClB,CAAC;YAEF,0CAA0C;YAC1C,IAAI,WAAyC,CAAC;YAC9C,IAAI,mBAAmB,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9F,WAAW,GAAG,2DAA4B,CAAC,KAAK,CAAC;gBACjD,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACvC,CAAC;iBAAM,IACL,mBAAmB,CAAC,IAAI,CACtB,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,GAAG;gBAClD,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,QAAQ;gBACvD,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,OAAO,CACjE,EACD,CAAC;gBACD,WAAW,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAEpD,MAAM,aAAa,GAAa,mBAAmB,CAAC,MAAM,CACxD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,QAAQ,CAClE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAChC,MAAM,YAAY,GAAa,mBAAmB,CAAC,MAAM,CACvD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,OAAO,CACjE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAChC,MAAM,QAAQ,GAAa,mBAAmB,CAAC,MAAM,CACnD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,GAAG,CAC7D,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAChC;;;;;;;;4BAQY;gBAEZ,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,oBAAoB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrH,CAAC;gBACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,eAAe,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACtG,CAAC;gBACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,QAAQ,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,mBAAmB,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,WAAW,GAAG,2DAA4B,CAAC,GAAG,CAAC;gBAC/C,MAAM,aAAa,GAAW,mBAAmB,CAAC,MAAM,CACtD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,2DAA4B,CAAC,GAAG,CAC7D,CAAC,MAAM,CAAC;gBACT,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,YAAY,CAAC,CAAC;YAC9C,CAAC;YAED,gEAAgE;YAChE,IAAI,cAA4C,CAAC;YACjD,IAAI,0BAA0B,CAAC,MAAM,KAAK,2DAA4B,CAAC,GAAG,EAAE,CAAC;gBAC3E,cAAc,GAAG,2DAA4B,CAAC,GAAG,CAAC;YACpD,CAAC;iBAAM,IAAI,0BAA0B,CAAC,MAAM,KAAK,2DAA4B,CAAC,QAAQ,EAAE,CAAC;gBACvF,cAAc,GAAG,2DAA4B,CAAC,QAAQ,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACN,IAAI,WAAW,KAAK,2DAA4B,CAAC,KAAK,EAAE,CAAC;oBACvD,cAAc,GAAG,2DAA4B,CAAC,KAAK,CAAC;gBACtD,CAAC;gBAAC,CAAC;oBACD,cAAc,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBACzD,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAA2G;gBACrH,GAAG,0BAA0B;gBAC7B,MAAM,EAAE,cAAc;gBAEtB,WAAW,EAAE,WAAW;gBACxB,WAAW,EAAE,WAAW;gBAExB,kBAAkB,EAAE,mBAAmB;aACxC,CAAC;YAEF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,KAAK,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,WAAuB,EAAE,WAAuB;QAGzE,IAAI,CAAC;YACH,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,uBAAU,CAAC;oBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,sBAAsB,EAAE,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;oBAC7F,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,GAAiC,2DAA4B,CAAC,KAAK,CAAC;YAC9E,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,wBAAwB;YACxB,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,EAAE,CAAC;gBAC1C,MAAM,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC/C,QAAQ,CAAC,IAAI,CAAC,+BAA+B,WAAW,CAAC,IAAI,SAAS,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC;YAC7F,CAAC;YAED,6BAA6B;YAC7B,IAAI,WAAW,CAAC,kBAAkB,KAAK,WAAW,CAAC,kBAAkB,EAAE,CAAC;gBACtE,MAAM,GAAG,2DAA4B,CAAC,QAAQ,CAAC;gBAC/C,QAAQ,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACvD,CAAC;YAED,sCAAsC;YACtC,IAAI,WAAW,CAAC,gBAAgB,IAAI,WAAW,CAAC,gBAAgB,EAAE,CAAC;gBACjE,MAAM,cAAc,GAAG,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxG,MAAM,cAAc,GAAG,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAExG,IAAI,cAAc,KAAK,cAAc,EAAE,CAAC;oBACtC,MAAM,GAAG,2DAA4B,CAAC,QAAQ,CAAC;oBAC/C,QAAQ,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC;YAED,IAAI,MAAM,KAAK,2DAA4B,CAAC,KAAK,EAAE,CAAC;gBAClD,QAAQ,CAAC,IAAI,CAAC,kCAAkC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,WAAW,GAA2G;gBAC1H,MAAM,EAAE,MAAM;gBAEd,WAAW,EAAE,WAAW;gBACxB,WAAW,EAAE,WAAW;gBAExB,kBAAkB,EAAE,EAAE;gBACtB,OAAO,EAAE,QAAQ;aAClB,CAAA;YAED,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,yBAAyB,EAAE,KAAK,CAAC;aAClE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,2BAA2B,CAC/B,qBAA6H;QAE7H,IAAI,CAAC;YACH,qBAAQ,CAAC,GAAG,CACV,4BACE,qBAAqB,EAAE,WAAW,EAAE,IAAI,IAAI,qBAAqB,EAAE,WAAW,EAAE,IAAI,IAAI,SAC1F,GAAG,CACJ,CAAC;YAEF,QAAQ,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBACrC,KAAK,2DAA4B,CAAC,GAAG;oBACnC,qBAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAClC,MAAM;gBAER,KAAK,2DAA4B,CAAC,KAAK;oBACrC,qBAAQ,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;oBACxC,MAAM;gBAER,KAAK,2DAA4B,CAAC,QAAQ;oBACxC,qBAAQ,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;oBACxC,MAAM;gBAER,KAAK,2DAA4B,CAAC,OAAO;oBACvC,qBAAQ,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;oBACtC,MAAM;YACV,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,qBAAqB,CAAC,OAAO,EAAE,CAAC;gBACnD,qBAAQ,CAAC,GAAG,CAAC,OAAO,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;YAED,qBAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,0BAA0B,EAAE,KAAK,CAAC;aACnE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF;AAjaD,sEAiaC"}

package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.d.ts

@@ -0,0 +1,118 @@
+import { DyFM_DataModel_Params, DyFM_DataProperty_Params, DyFM_DBFilter, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
+import { DyNTS_OAI_Embedding_ControlService } from '../oai-embedding.control-service';
+/**
+ * Vector Data Service that is connected to the relevant DBServices
+ * extends {@link DyNTS_DataService}
+ *
+ * For the vectorization, you need to setup the index in the DB to something like this:
+ * (the path declared here must be the same as the vectorizeUseIndex in the {@link DyFM_DataProperty_Params})
+ * ```json
+ * {
+ *   "fields": [
+ *     {
+ *       "type": "vector",
+ *       "path": "contentVectorized",
+ *       "numDimensions": 3072,
+ *       "similarity": "cosine"
+ *     }
+ *   ]
+ * }
+ * ```
+ *
+ * @example
+ * export class UserMatchStatisticsService extends DyNTS_DataService<UserMatchStatistics> {
+ *   constructor(
+ *     set: {
+ *       data?: UserMatchStatistics,
+ *       openAISettings?: DyFM_OAI_Settings,
+ *       issuer: string,
+ *     }
+ *   ) {
+ *     super(
+ *       new UserMatchStatistics(set?.data),
+ *       userMatchStatisticsModelParams,
+ *       set?.openAISettings ?? defaultOpenAISettings,
+ *       set.issuer,
+ *     );
+ *   }
+ * }
+ */
+export declare class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DataService<T> {
+    /**
+     * OpenAI settings
+     */
+    readonly openAISettings: DyFM_OAI_Settings;
+    /**
+     * Properties that are vectorized
+     * (properties that are marked with vectorizeFrom in other properties)
+     */
+    readonly vectorizeProperties: DyFM_DataProperty_Params<any, T>[];
+    readonly vectorizedProperties: DyFM_DataProperty_Params<any, T>[];
+    debugLog: boolean;
+    swallowVectorizedProperties: boolean;
+    embedding_CS: DyNTS_OAI_Embedding_ControlService;
+    constructor(
+    /**
+     * Initial data, this will be used by functions on default
+     */
+    data: T, 
+    /**
+     * DB data prams will be used to connect to usable dbService on GlobalService
+     */
+    dataParams: DyFM_DataModel_Params<T>, 
+    /**
+     * OpenAI settings
+     */
+    openAISettings: DyFM_OAI_Settings, 
+    /**
+     * Initial set for issuer to be able to follow the issuer's activity
+     */
+    issuer: string);
+    /**
+     * Vectorizes the properties of the data
+     * IF they are not vectorized yet OR alwaysVectorize is true
+     * (That are marked with vectorizeFrom if the specific property is not defined)
+     *
+     * @example
+     * const data = await vectorizeDataProperties(data);
+     *
+     * @returns The vectorized data
+     */
+    vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T>;
+    private checkAndVectorizeSingleProperty;
+    private getVectorizingDataForVectorizedProperty;
+    /**
+     * Vectorizes the input
+     *
+     * @example
+     * const vector = await vectorize(input);
+     */
+    vectorize(input: string, useModel?: DyFM_OAI_Model | string): Promise<number[]>;
+    /**
+     * Vectorizes the input and searches for the most similar data
+     * (the default limit is 3, and the default numberOfCandidates is 100)
+     *
+     * @example
+     * const result = await vectorSearch(input, searchInKey);
+     */
+    vectorSearch(set: {
+        input: string;
+        /** this should be the vectorized property key */
+        searchInKey: string;
+        limit?: number;
+        /**
+         * Number of candidates that are used to find the best match
+         */
+        numberOfCandidates?: number;
+        filterBy?: DyFM_DBFilter<T>;
+    }): Promise<any>;
+    /**
+     * VectorDataService overrides the saveData function to vectorize the data properties
+     *
+     * {@link DyNTS_DataService.saveData}
+     */
+    saveData(data?: T, dontSetToService?: boolean, forceVectorize?: boolean): Promise<T>;
+}
+//# sourceMappingURL=oai-vector-data.service.d.ts.map

package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oai-vector-data.service.d.ts","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,qBAAqB,EACrB,wBAAwB,EACxB,aAAa,EAEb,aAAa,EAEd,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAA2B,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAG9G,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAClF,OAAO,EAAE,kCAAkC,EAAE,MAAM,kCAAkC,CAAC;AAGtF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,2BAA2B,CAAC,CAAC,SAAS,aAAa,CAAE,SAAQ,iBAAiB,CAAC,CAAC,CAAC;IAwB1F;;OAEG;aACa,cAAc,EAAE,iBAAiB;IAzBnD;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,EAAE,wBAAwB,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;IACjE,QAAQ,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;IAGlE,QAAQ,EAAE,OAAO,CAAS;IAC1B,2BAA2B,EAAE,OAAO,CAAQ;IAE5C,YAAY,EAAE,kCAAkC,CAAC;;IAG/C;;OAEG;IACH,IAAI,EAAE,CAAC;IACP;;OAEG;IACH,UAAU,EAAE,qBAAqB,CAAC,CAAC,CAAC;IACpC;;OAEG;IACa,cAAc,EAAE,iBAAiB;IACjD;;OAEG;IACH,MAAM,EAAE,MAAM;IA6FhB;;;;;;;;;OASG;IACG,uBAAuB,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,cAAc,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;YAuBlE,+BAA+B;YAkD/B,uCAAuC;IA8BrD;;;;;OAKG;IACG,SAAS,CACb,KAAK,EAAE,MAAM,EACb,QAAQ,GAAE,cAAc,GAAG,MAAqD,GAC/E,OAAO,CAAC,MAAM,EAAE,CAAC;IA4BpB;;;;;;OAMG;IACG,YAAY,CAChB,GAAG,EAAE;QACH,KAAK,EAAE,MAAM,CAAC;QACd,iDAAiD;QACjD,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;WAEG;QACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;KAC7B,GACA,OAAO,CAAC,GAAG,CAAC;IAyFf;;;;OAIG;IACY,QAAQ,CACrB,IAAI,CAAC,EAAE,CAAC,EACR,gBAAgB,CAAC,EAAE,OAAO,EAC1B,cAAc,CAAC,EAAE,OAAO,GACvB,OAAO,CAAC,CAAC,CAAC;CA4Bd"}