@carlonicora/nestjs-neo4jsonapi 0.0.1

package/dist/foundations/chunker/services/chunker.service.d.ts

@@ -0,0 +1,36 @@
+import { Document } from "@langchain/core/documents";
+import { ModelService } from "../../../core/llm/services/model.service";
+import { DocXService } from "../../chunker/services/types/docx.service";
+import { PdfService } from "../../chunker/services/types/pdf.service";
+import { PptxService } from "../../chunker/services/types/pptx.service";
+import { SemanticSplitterService } from "../../chunker/services/types/semanticsplitter.service";
+import { XlsxService } from "../../chunker/services/types/xlsx.service";
+import { S3Service } from "../../s3/services/s3.service";
+export declare class ChunkerService {
+    private readonly semanticSplitterService;
+    private readonly docxService;
+    private readonly pptxService;
+    private readonly pdfService;
+    private readonly xlsxService;
+    private readonly s3Service;
+    private readonly modelService;
+    private logger;
+    constructor(semanticSplitterService: SemanticSplitterService, docxService: DocXService, pptxService: PptxService, pdfService: PdfService, xlsxService: XlsxService, s3Service: S3Service, modelService: ModelService);
+    private _downloadFileAsBuffer;
+    private _downloadFile;
+    generateContentStructureFromFile(params: {
+        fileType: string;
+        filePath: string;
+    }): Promise<Document[]>;
+    generateContentStructureFromMarkdown(params: {
+        content: string;
+        title?: string;
+    }): Promise<Document[]>;
+    private _createChunksFromImage;
+    private _createFromMarkdown;
+    private _createFromDocX;
+    private _createFromPptx;
+    private _createFromXlsx;
+    private _createFromPdf;
+}
+//# sourceMappingURL=chunker.service.d.ts.map

package/dist/foundations/chunker/services/chunker.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.service.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunker/services/chunker.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAMrD,OAAO,EAAE,YAAY,EAAE,MAAM,0CAA0C,CAAC;AAExE,OAAO,EAAE,WAAW,EAAE,MAAM,2CAA2C,CAAC;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,0CAA0C,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,2CAA2C,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uDAAuD,CAAC;AAChG,OAAO,EAAE,WAAW,EAAE,MAAM,2CAA2C,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAIzD,qBACa,cAAc;IAIvB,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAT/B,OAAO,CAAC,MAAM,CAA2C;gBAGtC,uBAAuB,EAAE,uBAAuB,EAChD,WAAW,EAAE,WAAW,EACxB,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,SAAS,EACpB,YAAY,EAAE,YAAY;YAG/B,qBAAqB;YAQrB,aAAa;IAOrB,gCAAgC,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA2DrG,oCAAoC,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;YAuB9F,sBAAsB;YA+CtB,mBAAmB;YAsBnB,eAAe;YAuCf,eAAe;YAoCf,eAAe;YAmCf,cAAc;CAgD7B"}

package/dist/foundations/chunker/services/chunker.service.js

@@ -0,0 +1,371 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var ChunkerService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ChunkerService = void 0;
+const csv_1 = require("@langchain/community/document_loaders/fs/csv");
+const documents_1 = require("@langchain/core/documents");
+const messages_1 = require("@langchain/core/messages");
+const textsplitters_1 = require("@langchain/textsplitters");
+const common_1 = require("@nestjs/common");
+const crypto_1 = require("crypto");
+const fs = __importStar(require("fs/promises"));
+const model_service_1 = require("../../../core/llm/services/model.service");
+const file_types_1 = require("../../chunker/constants/file.types");
+const docx_service_1 = require("../../chunker/services/types/docx.service");
+const pdf_service_1 = require("../../chunker/services/types/pdf.service");
+const pptx_service_1 = require("../../chunker/services/types/pptx.service");
+const semanticsplitter_service_1 = require("../../chunker/services/types/semanticsplitter.service");
+const xlsx_service_1 = require("../../chunker/services/types/xlsx.service");
+const s3_service_1 = require("../../s3/services/s3.service");
+const json_loader_1 = require("../loaders/json.loader");
+const text_loader_1 = require("../loaders/text.loader");
+let ChunkerService = ChunkerService_1 = class ChunkerService {
+    constructor(semanticSplitterService, docxService, pptxService, pdfService, xlsxService, s3Service, modelService) {
+        this.semanticSplitterService = semanticSplitterService;
+        this.docxService = docxService;
+        this.pptxService = pptxService;
+        this.pdfService = pdfService;
+        this.xlsxService = xlsxService;
+        this.s3Service = s3Service;
+        this.modelService = modelService;
+        this.logger = new common_1.Logger(ChunkerService_1.name);
+    }
+    async _downloadFileAsBuffer(params) {
+        const response = await fetch(params.url);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch the file: ${response.statusText}`);
+        }
+        return Buffer.from(await response.arrayBuffer());
+    }
+    async _downloadFile(params) {
+        const buffer = await this._downloadFileAsBuffer(params);
+        const tempFilePath = `/tmp/temp-file.${(0, crypto_1.randomUUID)()}.${params.extension}`;
+        await fs.writeFile(tempFilePath, buffer);
+        return tempFilePath;
+    }
+    async generateContentStructureFromFile(params) {
+        if ((0, file_types_1.isImageFile)(params.fileType))
+            return this._createChunksFromImage(params);
+        const localFilePath = await this._downloadFile({
+            url: params.filePath,
+            extension: params.fileType,
+        });
+        let response = [];
+        switch (params.fileType.toLowerCase()) {
+            case "md":
+                response = await this._createFromMarkdown({ filePath: params.filePath, localFilePath });
+                break;
+            case "docx":
+                response = await this._createFromDocX({ filePath: params.filePath, localFilePath });
+                break;
+            case "pptx":
+            case "presentation":
+                response = await this._createFromPptx({ filePath: params.filePath, localFilePath });
+                break;
+            case "xlsx":
+            case "spreadsheet":
+                response = await this._createFromXlsx({ filePath: params.filePath, localFilePath });
+                break;
+            case "pdf":
+                response = await this._createFromPdf({ filePath: params.filePath, localFilePath });
+                break;
+            default:
+                let loader;
+                switch (params.fileType.toLowerCase()) {
+                    case "json":
+                        loader = new json_loader_1.JSONLoader(localFilePath, "/texts");
+                        break;
+                    case "jsonl":
+                        loader = new json_loader_1.JSONLinesLoader(localFilePath, "/html");
+                        break;
+                    case "csv":
+                        loader = new csv_1.CSVLoader(localFilePath, "text");
+                        break;
+                    default:
+                        loader = new text_loader_1.TextLoader(localFilePath);
+                        break;
+                }
+                const rawDocs = await loader.load();
+                const textSplitter = new textsplitters_1.TokenTextSplitter({
+                    chunkSize: 1000,
+                    chunkOverlap: 200,
+                });
+                response = await textSplitter.splitDocuments(rawDocs);
+                break;
+        }
+        return response;
+    }
+    async generateContentStructureFromMarkdown(params) {
+        try {
+            const semanticChunks = await this.semanticSplitterService.splitMarkdownToChunks({
+                content: params.content,
+                title: params.title,
+            });
+            if (semanticChunks && semanticChunks.length > 0) {
+                return semanticChunks;
+            }
+        }
+        catch (error) {
+            this.logger.warn("Semantic markdown splitting failed, falling back to markdown splitter:", error);
+        }
+        const splitter = new textsplitters_1.MarkdownTextSplitter({
+            chunkSize: 1500,
+            chunkOverlap: 200,
+        });
+        const response = await splitter.createDocuments([params.content]);
+        return response;
+    }
+    async _createChunksFromImage(params) {
+        try {
+            const imageUrl = params.filePath.toLowerCase().startsWith("http")
+                ? params.filePath
+                : await this.s3Service.generateSignedUrl({ key: params.filePath });
+            const imageBuffer = await this._downloadFileAsBuffer({
+                url: imageUrl,
+                extension: params.fileType,
+            });
+            const model = this.modelService.getLLM({ temperature: 0.2 });
+            // Analyze the image directly using the LLM
+            const imageDescription = await model.invoke([
+                new messages_1.HumanMessage({
+                    content: [
+                        {
+                            type: "text",
+                            text: "Describe this image in detail, including all visible text, objects, people, scenes, and any relevant information that would be useful for document processing and knowledge extraction.",
+                        },
+                        {
+                            type: "image_url",
+                            image_url: {
+                                url: `data:image/${params.fileType};base64,${imageBuffer.toString("base64")}`,
+                            },
+                        },
+                    ],
+                }),
+            ]);
+            if (imageDescription?.content) {
+                // Create chunks from the image description
+                const splitter = new textsplitters_1.RecursiveCharacterTextSplitter({
+                    chunkSize: 1000,
+                    chunkOverlap: 200,
+                });
+                return await splitter.createDocuments([imageDescription.content.toString()]);
+            }
+            return [];
+        }
+        catch (error) {
+            this.logger.error("Error processing image file:", error);
+            return [];
+        }
+    }
+    async _createFromMarkdown(params) {
+        const markdown = await fs.readFile(params.localFilePath, "utf-8");
+        const urlParts = params.filePath.split("/");
+        const filename = urlParts[urlParts.length - 1];
+        const title = filename.replace(/\.md$/i, "");
+        try {
+            return await this.semanticSplitterService.splitMarkdownToChunks({
+                content: markdown,
+                title: title,
+            });
+        }
+        catch (error) {
+            this.logger.error("Semantic markdown splitting failed, falling back to markdown splitter:", error);
+            const splitter = new textsplitters_1.MarkdownTextSplitter({
+                chunkSize: 1500,
+                chunkOverlap: 200,
+            });
+            return await splitter.createDocuments([markdown]);
+        }
+    }
+    async _createFromDocX(params) {
+        const rawElements = await this.docxService.getRawElements(params.localFilePath);
+        const markdownContent = this.docxService.convertToMarkdown(rawElements);
+        if (markdownContent && markdownContent.trim()) {
+            try {
+                return await this.semanticSplitterService.splitMarkdownToChunks({
+                    content: markdownContent,
+                });
+            }
+            catch (error) {
+                this.logger.error("Semantic markdown splitting failed, falling back to markdown splitter:", error);
+                const splitter = new textsplitters_1.MarkdownTextSplitter({
+                    chunkSize: 1500,
+                    chunkOverlap: 200,
+                });
+                return await splitter.createDocuments([markdownContent]);
+            }
+        }
+        const response = [];
+        const documentParts = await this.docxService.load({ filePath: params.localFilePath });
+        for (const part of documentParts) {
+            if (part.metadata?.type === "paragraphs") {
+                const splitter = new textsplitters_1.RecursiveCharacterTextSplitter({
+                    chunkSize: 500,
+                    chunkOverlap: 20,
+                });
+                const parts = await splitter.createDocuments([part.pageContent]);
+                response.push(...parts);
+            }
+            else {
+                response.push(part);
+            }
+        }
+        return response;
+    }
+    async _createFromPptx(params) {
+        const extractedContent = await this.pptxService.getRawElements(params.localFilePath);
+        const markdownContent = this.pptxService.convertToMarkdown(extractedContent);
+        if (markdownContent && markdownContent.trim()) {
+            try {
+                return await this.semanticSplitterService.splitMarkdownToChunks({
+                    content: markdownContent,
+                    title: undefined,
+                });
+            }
+            catch (error) {
+                this.logger.error("Presentation processing failed:", error);
+                const splitter = new textsplitters_1.MarkdownTextSplitter({
+                    chunkSize: 1500,
+                    chunkOverlap: 200,
+                });
+                return await splitter.createDocuments([markdownContent]);
+            }
+        }
+        const response = [];
+        const documentParts = await this.pptxService.load({ filePath: params.localFilePath });
+        for (const part of documentParts) {
+            const splitter = new textsplitters_1.RecursiveCharacterTextSplitter({
+                chunkSize: 800,
+                chunkOverlap: 100,
+            });
+            const parts = await splitter.createDocuments([part.pageContent]);
+            response.push(...parts);
+        }
+        return response;
+    }
+    async _createFromXlsx(params) {
+        const worksheetData = await this.xlsxService.extractXlsxContent(params.localFilePath);
+        if (worksheetData && worksheetData.length > 0) {
+            const markdownChunks = this.xlsxService.convertToMarkdownChunks(worksheetData);
+            return markdownChunks.map((chunkContent, index) => new documents_1.Document({
+                pageContent: chunkContent,
+                metadata: {
+                    type: "xlsx",
+                    source: params.filePath,
+                    chunkIndex: index,
+                    totalChunks: markdownChunks.length,
+                },
+            }));
+        }
+        const response = [];
+        const documentParts = await this.xlsxService.load({ filePath: params.localFilePath });
+        for (const part of documentParts) {
+            const splitter = new textsplitters_1.RecursiveCharacterTextSplitter({
+                chunkSize: 1000,
+                chunkOverlap: 100,
+            });
+            const parts = await splitter.createDocuments([part.pageContent]);
+            response.push(...parts);
+        }
+        return response;
+    }
+    async _createFromPdf(params) {
+        try {
+            const pdfContent = await this.pdfService.extractPdfContent(params.localFilePath);
+            const markdownContent = pdfContent
+                .map((block) => {
+                if (block.type === "header") {
+                    return `# ${block.content}`;
+                }
+                else if (block.type === "table") {
+                    return typeof block.content === "string" ? block.content : JSON.stringify(block.content);
+                }
+                else {
+                    return block.content;
+                }
+            })
+                .join("\n\n");
+            if (markdownContent && markdownContent.trim()) {
+                try {
+                    return await this.semanticSplitterService.splitMarkdownToChunks({
+                        content: markdownContent,
+                        title: undefined,
+                    });
+                }
+                catch (error) {
+                    this.logger.error("Presentation processing failed:", error);
+                    const splitter = new textsplitters_1.MarkdownTextSplitter({
+                        chunkSize: 1500,
+                        chunkOverlap: 200,
+                    });
+                    return await splitter.createDocuments([markdownContent]);
+                }
+            }
+            const rawElements = await this.pdfService.getRawElements(params.localFilePath);
+            return rawElements.map((element) => new documents_1.Document({
+                pageContent: element.content,
+                metadata: {
+                    type: element.type,
+                },
+            }));
+        }
+        catch (error) {
+            this.logger.error("PDF processing failed:", error);
+            return [];
+        }
+    }
+};
+exports.ChunkerService = ChunkerService;
+exports.ChunkerService = ChunkerService = ChunkerService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [semanticsplitter_service_1.SemanticSplitterService,
+        docx_service_1.DocXService,
+        pptx_service_1.PptxService,
+        pdf_service_1.PdfService,
+        xlsx_service_1.XlsxService,
+        s3_service_1.S3Service,
+        model_service_1.ModelService])
+], ChunkerService);
+//# sourceMappingURL=chunker.service.js.map

package/dist/foundations/chunker/services/chunker.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.service.js","sourceRoot":"","sources":["../../../../src/foundations/chunker/services/chunker.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sEAAyE;AAEzE,yDAAqD;AACrD,uDAAwD;AACxD,4DAAmH;AACnH,2CAAoD;AACpD,mCAAoC;AACpC,gDAAkC;AAClC,4EAAwE;AACxE,mEAAiE;AACjE,4EAAwE;AACxE,0EAAsE;AACtE,4EAAwE;AACxE,oGAAgG;AAChG,4EAAwE;AACxE,6DAAyD;AACzD,wDAAqE;AACrE,wDAAoD;AAG7C,IAAM,cAAc,sBAApB,MAAM,cAAc;IAGzB,YACmB,uBAAgD,EAChD,WAAwB,EACxB,WAAwB,EACxB,UAAsB,EACtB,WAAwB,EACxB,SAAoB,EACpB,YAA0B;QAN1B,4BAAuB,GAAvB,uBAAuB,CAAyB;QAChD,gBAAW,GAAX,WAAW,CAAa;QACxB,gBAAW,GAAX,WAAW,CAAa;QACxB,eAAU,GAAV,UAAU,CAAY;QACtB,gBAAW,GAAX,WAAW,CAAa;QACxB,cAAS,GAAT,SAAS,CAAW;QACpB,iBAAY,GAAZ,YAAY,CAAc;QATrC,WAAM,GAAW,IAAI,eAAM,CAAC,gBAAc,CAAC,IAAI,CAAC,CAAC;IAUtD,CAAC;IAEI,KAAK,CAAC,qBAAqB,CAAC,MAA0C;QAC5E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACnD,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAA0C;QACpE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,YAAY,GAAG,kBAAkB,IAAA,mBAAU,GAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1E,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gCAAgC,CAAC,MAA8C;QACnF,IAAI,IAAA,wBAAW,EAAC,MAAM,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAE7E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC;YAC7C,GAAG,EAAE,MAAM,CAAC,QAAQ;YACpB,SAAS,EAAE,MAAM,CAAC,QAAQ;SAC3B,CAAC,CAAC;QAEH,IAAI,QAAQ,GAAe,EAAE,CAAC;QAE9B,QAAQ,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,KAAK,IAAI;gBACP,QAAQ,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;gBACxF,MAAM;YACR,KAAK,MAAM;gBACT,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;gBACpF,MAAM;YACR,KAAK,MAAM,CAAC;YACZ,KAAK,cAAc;gBACjB,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;gBACpF,MAAM;YACR,KAAK,MAAM,CAAC;YACZ,KAAK,aAAa;gBAChB,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;gBACpF,MAAM;YACR,KAAK,KAAK;gBACR,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;gBACnF,MAAM;YACR;gBACE,IAAI,MAA0B,CAAC;gBAC/B,QAAQ,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;oBACtC,KAAK,MAAM;wBACT,MAAM,GAAG,IAAI,wBAAU,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;wBACjD,MAAM;oBACR,KAAK,OAAO;wBACV,MAAM,GAAG,IAAI,6BAAe,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;wBACrD,MAAM;oBACR,KAAK,KAAK;wBACR,MAAM,GAAG,IAAI,eAAS,CAAC,aAAa,EAAE,MAAM,CAAQ,CAAC;wBACrD,MAAM;oBACR;wBACE,MAAM,GAAG,IAAI,wBAAU,CAAC,aAAa,CAAC,CAAC;wBACvC,MAAM;gBACV,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEpC,MAAM,YAAY,GAAG,IAAI,iCAAiB,CAAC;oBACzC,SAAS,EAAE,IAAI;oBACf,YAAY,EAAE,GAAG;iBAClB,CAAC,CAAC;gBAEH,QAAQ,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;gBACtD,MAAM;QACV,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,oCAAoC,CAAC,MAA2C;QACpF,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,qBAAqB,CAAC;gBAC9E,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,IAAI,cAAc,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChD,OAAO,cAAc,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wEAAwE,EAAE,KAAK,CAAC,CAAC;QACpG,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,oCAAoB,CAAC;YACxC,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,GAAG;SAClB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;QAElE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,MAA8C;QACjF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC/D,CAAC,CAAC,MAAM,CAAC,QAAQ;gBACjB,CAAC,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAErE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;gBACnD,GAAG,EAAE,QAAQ;gBACb,SAAS,EAAE,MAAM,CAAC,QAAQ;aAC3B,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,2CAA2C;YAC3C,MAAM,gBAAgB,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;gBAC1C,IAAI,uBAAY,CAAC;oBACf,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,yLAAyL;yBAChM;wBACD;4BACE,IAAI,EAAE,WAAW;4BACjB,SAAS,EAAE;gCACT,GAAG,EAAE,cAAc,MAAM,CAAC,QAAQ,WAAW,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;6BAC9E;yBACF;qBACF;iBACF,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,OAAO,EAAE,CAAC;gBAC9B,2CAA2C;gBAC3C,MAAM,QAAQ,GAAG,IAAI,8CAA8B,CAAC;oBAClD,SAAS,EAAE,IAAI;oBACf,YAAY,EAAE,GAAG;iBAClB,CAAC,CAAC;gBACH,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC/E,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;YACzD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAmD;QACnF,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAElE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE7C,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,uBAAuB,CAAC,qBAAqB,CAAC;gBAC9D,OAAO,EAAE,QAAQ;gBACjB,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wEAAwE,EAAE,KAAK,CAAC,CAAC;YACnG,MAAM,QAAQ,GAAG,IAAI,oCAAoB,CAAC;gBACxC,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,GAAG;aAClB,CAAC,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,MAAmD;QAC/E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAChF,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAExE,IAAI,eAAe,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,uBAAuB,CAAC,qBAAqB,CAAC;oBAC9D,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wEAAwE,EAAE,KAAK,CAAC,CAAC;gBACnG,MAAM,QAAQ,GAAG,IAAI,oCAAoB,CAAC;oBACxC,SAAS,EAAE,IAAI;oBACf,YAAY,EAAE,GAAG;iBAClB,CAAC,CAAC;gBACH,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;QAEtF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,QAAQ,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,IAAI,8CAA8B,CAAC;oBAClD,SAAS,EAAE,GAAG;oBACd,YAAY,EAAE,EAAE;iBACjB,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBAEjE,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,MAAmD;QAC/E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAErF,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE7E,IAAI,eAAe,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,uBAAuB,CAAC,qBAAqB,CAAC;oBAC9D,OAAO,EAAE,eAAe;oBACxB,KAAK,EAAE,SAAS;iBACjB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,QAAQ,GAAG,IAAI,oCAAoB,CAAC;oBACxC,SAAS,EAAE,IAAI;oBACf,YAAY,EAAE,GAAG;iBAClB,CAAC,CAAC;gBACH,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;QAEtF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,8CAA8B,CAAC;gBAClD,SAAS,EAAE,GAAG;gBACd,YAAY,EAAE,GAAG;aAClB,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACjE,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QAC1B,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,MAAmD;QAC/E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAEtF,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,aAAa,CAAC,CAAC;YAE/E,OAAO,cAAc,CAAC,GAAG,CACvB,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE,CACtB,IAAI,oBAAQ,CAAC;gBACX,WAAW,EAAE,YAAY;gBACzB,QAAQ,EAAE;oBACR,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,MAAM,CAAC,QAAQ;oBACvB,UAAU,EAAE,KAAK;oBACjB,WAAW,EAAE,cAAc,CAAC,MAAM;iBACnC;aACF,CAAC,CACL,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;QAEtF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,8CAA8B,CAAC;gBAClD,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,GAAG;aAClB,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACjE,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QAC1B,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,MAAmD;QAC9E,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAEjF,MAAM,eAAe,GAAG,UAAU;iBAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACb,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC5B,OAAO,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9B,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAClC,OAAO,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC3F,CAAC;qBAAM,CAAC;oBACN,OAAO,KAAK,CAAC,OAAO,CAAC;gBACvB,CAAC;YACH,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,IAAI,eAAe,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9C,IAAI,CAAC;oBACH,OAAO,MAAM,IAAI,CAAC,uBAAuB,CAAC,qBAAqB,CAAC;wBAC9D,OAAO,EAAE,eAAe;wBACxB,KAAK,EAAE,SAAS;qBACjB,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;oBAC5D,MAAM,QAAQ,GAAG,IAAI,oCAAoB,CAAC;wBACxC,SAAS,EAAE,IAAI;wBACf,YAAY,EAAE,GAAG;qBAClB,CAAC,CAAC;oBACH,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAE/E,OAAO,WAAW,CAAC,GAAG,CACpB,CAAC,OAAO,EAAE,EAAE,CACV,IAAI,oBAAQ,CAAC;gBACX,WAAW,EAAE,OAAO,CAAC,OAAO;gBAC5B,QAAQ,EAAE;oBACR,IAAI,EAAE,OAAO,CAAC,IAAI;iBACnB;aACF,CAAC,CACL,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;YACnD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF,CAAA;AAjVY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAKiC,kDAAuB;QACnC,0BAAW;QACX,0BAAW;QACZ,wBAAU;QACT,0BAAW;QACb,sBAAS;QACN,4BAAY;GAVlC,cAAc,CAiV1B"}

package/dist/foundations/chunker/services/types/docx.service.d.ts

@@ -0,0 +1,26 @@
+import { Document } from "@langchain/core/documents";
+type DocxContent = {
+    type: "table" | "paragraphs";
+    content: string | any;
+};
+export declare class DocXService {
+    private numberingState;
+    resetNumberingState(): void;
+    convertToMarkdown(docxElements: any[]): string;
+    private extractElement;
+    private extractTableAsMarkdown;
+    private extractCellContentAsMarkdown;
+    private checkForTableHeader;
+    private extractParagraph;
+    private extractListItem;
+    private extractLine;
+    private extractTable;
+    private extractCellContent;
+    extractDocxContent(docPath: string): Promise<DocxContent[]>;
+    getRawElements(filePath: string): Promise<any[]>;
+    load(params: {
+        filePath: string;
+    }): Promise<Document[]>;
+}
+export {};
+//# sourceMappingURL=docx.service.d.ts.map

package/dist/foundations/chunker/services/types/docx.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.service.d.ts","sourceRoot":"","sources":["../../../../../src/foundations/chunker/services/types/docx.service.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAGrD,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,OAAO,GAAG,YAAY,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC;CACvB,CAAC;AAEF,qBACa,WAAW;IACtB,OAAO,CAAC,cAAc,CAA8B;IAE7C,mBAAmB,IAAI,IAAI;IAI3B,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,GAAG,MAAM;IAQrD,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,sBAAsB;IAuF9B,OAAO,CAAC,4BAA4B;IAoBpC,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,CAAC,gBAAgB;IA4FxB,OAAO,CAAC,eAAe;IAsBvB,OAAO,CAAC,WAAW;IAwCnB,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,kBAAkB;IAoCpB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA8C3D,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IA2BhD,IAAI,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;CA0B9D"}