@magda/semantic-indexer-framework 5.2.0

package/dist/test/onRecordFoundStorageObject.spec.js

@@ -0,0 +1,490 @@
+import { expect } from "chai";
+import sinon from "sinon";
+import { onRecordFoundStorageObject } from "../onRecordFoundStorageObject.js";
+import { createRecord, expectNoThrowsAsync } from "./helpers.js";
+import { BaseSemanticIndexerTest } from "./BaseSemanticIndexerTest.js";
+import fs from "fs";
+import nock from "nock";
+import { tmpdir } from "os";
+import { join } from "path";
+describe("onRecordFoundStorageObject", () => {
+    let testEnv;
+    beforeEach(() => {
+        testEnv = new BaseSemanticIndexerTest({
+            suppressConsoleLogs: true
+        });
+    });
+    afterEach(() => {
+        testEnv.cleanup();
+    });
+    it("should handle and index storage object with expected formatType, with autoDownloadFile disabled", async () => {
+        testEnv.createEmbeddingTextStub.resolves({
+            text: "embedding text: test1"
+        });
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            autoDownloadFile: false
+        });
+        testEnv.chunker.chunk
+            .withArgs("embedding text: test1")
+            .returns([{ text: "test1", length: 5, position: 0, overlap: 0 }]);
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .resolves([[0.1, 0.2, 0.3]]);
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await onRecordFound(record, testEnv.registry);
+        const expectedDocs = [
+            {
+                itemType: userConfig.itemType,
+                recordId: "id1",
+                parentRecordId: testEnv.DEFAULT_PARENT_RECORD_ID,
+                fileFormat: "csv",
+                index_text_chunk: "test1",
+                embedding: [0.1, 0.2, 0.3],
+                only_one_index_text_chunk: true,
+                index_text_chunk_length: 5,
+                index_text_chunk_position: 0,
+                index_text_chunk_overlap: 0,
+                indexerId: userConfig.id,
+                createTime: testEnv.getCurrentTimeString(),
+                updateTime: testEnv.getCurrentTimeString()
+            }
+        ];
+        testEnv.expectSuccessCalls({
+            createEmbeddingTextCallCount: 1,
+            chunkCallCount: 1,
+            embeddingApiCallCount: 1,
+            bulkIndexCallCount: 1,
+            deleteByQueryCallCount: 1
+        });
+        testEnv.expectCalledWith(testEnv.createEmbeddingTextStub, 0, {
+            record,
+            format: "csv",
+            filePath: null,
+            url: "http://test.com/file.csv",
+            readonlyRegistry: testEnv.registry
+        });
+        testEnv.expectIndexedDocs(expectedDocs);
+    });
+    it("should handle and index storage object with expected formatType, with autoDownloadFile enabled", async () => {
+        const fileContent = "test1";
+        nock("http://test.com").get("/file.csv").reply(200, fileContent);
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        let usedFilePath = null;
+        testEnv.createEmbeddingTextStub.callsFake(async ({ record, format, filePath, url }) => {
+            expect(record).to.deep.equal(record);
+            expect(format).to.equal("csv");
+            expect(fs.existsSync(filePath)).to.be.true;
+            expect(url).to.equal("http://test.com/file.csv");
+            const content = fs.readFileSync(filePath, "utf-8");
+            expect(content).to.equal("test1");
+            usedFilePath = filePath;
+            return { text: "embedding text: test1" };
+        });
+        testEnv.chunker.chunk
+            .withArgs("embedding text: test1")
+            .returns([{ text: "test1", length: 5, position: 0, overlap: 0 }]);
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .resolves([[0.1, 0.2, 0.3]]);
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            autoDownloadFile: true
+        });
+        const expectedDocs = [
+            {
+                itemType: userConfig.itemType,
+                recordId: "id1",
+                parentRecordId: testEnv.DEFAULT_PARENT_RECORD_ID,
+                fileFormat: "csv",
+                index_text_chunk: "test1",
+                embedding: [0.1, 0.2, 0.3],
+                only_one_index_text_chunk: true,
+                index_text_chunk_length: 5,
+                index_text_chunk_position: 0,
+                index_text_chunk_overlap: 0,
+                indexerId: userConfig.id,
+                createTime: testEnv.getCurrentTimeString(),
+                updateTime: testEnv.getCurrentTimeString()
+            }
+        ];
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await onRecordFound(record, testEnv.registry);
+        testEnv.expectSuccessCalls({
+            createEmbeddingTextCallCount: 1,
+            chunkCallCount: 1,
+            embeddingApiCallCount: 1,
+            bulkIndexCallCount: 1,
+            deleteByQueryCallCount: 1
+        });
+        testEnv.expectIndexedDocs(expectedDocs);
+        if (usedFilePath) {
+            expect(fs.existsSync(usedFilePath)).to.be.false;
+        }
+        nock.cleanAll();
+    });
+    it("should successfully download file from minio", async () => {
+        const fileContent = "test1";
+        const downloadUrl = "magda://storage-api/fake-parent-record-id/id1/file.csv";
+        const tempDir = tmpdir();
+        const tempFilePath = join(tempDir, "file.csv");
+        testEnv.minioClient.downloadFile
+            .withArgs(downloadUrl)
+            .callsFake(async () => {
+            fs.writeFileSync(tempFilePath, fileContent);
+            return tempFilePath;
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: downloadUrl
+                }
+            }
+        });
+        let usedFilePath = null;
+        testEnv.createEmbeddingTextStub.callsFake(async ({ record, format, filePath, url }) => {
+            expect(record).to.deep.equal(record);
+            expect(format).to.equal("csv");
+            expect(fs.existsSync(filePath)).to.be.true;
+            expect(url).to.equal(downloadUrl);
+            const content = fs.readFileSync(filePath, "utf-8");
+            expect(content).to.equal("test1");
+            usedFilePath = filePath;
+            return { text: "embedding text: test1" };
+        });
+        testEnv.chunker.chunk
+            .withArgs("embedding text: test1")
+            .returns([{ text: "test1", length: 5, position: 0, overlap: 0 }]);
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .resolves([[0.1, 0.2, 0.3]]);
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            autoDownloadFile: true
+        });
+        const expectedDocs = [
+            {
+                itemType: userConfig.itemType,
+                recordId: "id1",
+                parentRecordId: testEnv.DEFAULT_PARENT_RECORD_ID,
+                fileFormat: "csv",
+                index_text_chunk: "test1",
+                embedding: [0.1, 0.2, 0.3],
+                only_one_index_text_chunk: true,
+                index_text_chunk_length: 5,
+                index_text_chunk_position: 0,
+                index_text_chunk_overlap: 0,
+                indexerId: userConfig.id,
+                createTime: testEnv.getCurrentTimeString(),
+                updateTime: testEnv.getCurrentTimeString()
+            }
+        ];
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await onRecordFound(record, testEnv.registry);
+        testEnv.expectSuccessCalls({
+            createEmbeddingTextCallCount: 1,
+            chunkCallCount: 1,
+            embeddingApiCallCount: 1,
+            bulkIndexCallCount: 1,
+            deleteByQueryCallCount: 1
+        });
+        testEnv.expectIndexedDocs(expectedDocs);
+        if (usedFilePath) {
+            expect(fs.existsSync(usedFilePath)).to.be.false;
+        }
+        nock.cleanAll();
+    });
+    it("should filter out records with unexpected formatType", async () => {
+        testEnv.createEmbeddingTextStub.resolves({
+            text: "embedding text: test1"
+        });
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            autoDownloadFile: false
+        });
+        testEnv.chunker.chunk
+            .withArgs("embedding text: test1")
+            .returns([{ text: "test1", length: 5, position: 0, overlap: 0 }]);
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .resolves([[0.1, 0.2, 0.3]]);
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "json" },
+                "dcat-distribution-strings": {
+                    format: "json",
+                    downloadURL: "http://test.com/file.json"
+                }
+            }
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await onRecordFound(record, testEnv.registry);
+        testEnv.expectSuccessCalls({
+            createEmbeddingTextCallCount: 0,
+            chunkCallCount: 0,
+            embeddingApiCallCount: 0,
+            bulkIndexCallCount: 0,
+            deleteByQueryCallCount: 0
+        });
+    });
+    it("should handle and index storage object with multiple expected format types", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv", "json"],
+            autoDownloadFile: false
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        testEnv.chunker.chunk
+            .withArgs("embedding text: test1")
+            .returns([{ text: "test1", length: 5, position: 0, overlap: 0 }]);
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .resolves([[0.1, 0.2, 0.3]]);
+        testEnv.createEmbeddingTextStub
+            .withArgs({
+            record,
+            format: "csv",
+            filePath: null,
+            url: "http://test.com/file.csv",
+            readonlyRegistry: testEnv.registry
+        })
+            .resolves({ text: "embedding text: test1" });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await onRecordFound(record, testEnv.registry);
+        const expectedDocs1 = [
+            {
+                itemType: userConfig.itemType,
+                recordId: "id1",
+                parentRecordId: testEnv.DEFAULT_PARENT_RECORD_ID,
+                fileFormat: "csv",
+                index_text_chunk: "test1",
+                embedding: [0.1, 0.2, 0.3],
+                only_one_index_text_chunk: true,
+                index_text_chunk_length: 5,
+                index_text_chunk_position: 0,
+                index_text_chunk_overlap: 0,
+                indexerId: userConfig.id,
+                createTime: testEnv.getCurrentTimeString(),
+                updateTime: testEnv.getCurrentTimeString()
+            }
+        ];
+        testEnv.expectSuccessCalls({
+            createEmbeddingTextCallCount: 1,
+            chunkCallCount: 1,
+            embeddingApiCallCount: 1,
+            bulkIndexCallCount: 1,
+            deleteByQueryCallCount: 1
+        });
+        testEnv.expectIndexedDocs(expectedDocs1);
+    });
+    it("should skip this record when embeddingApiClient throws an error", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"]
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .rejects(new Error("throw test error"));
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
+    });
+    it("should skip this record when opensearchApiClient throws an error", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"]
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        testEnv.opensearchApiClient.bulkIndexDocument
+            .withArgs(record, null)
+            .rejects(new Error("throw test error"));
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
+    });
+    it("should skip this record when createEmbeddingText throws an error", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            createEmbeddingText: sinon
+                .stub()
+                .rejects(new Error("throw test error"))
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
+    });
+    it("should skip this record when dataset-distributions aspect not exist", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"]
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {}
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
+    });
+    it("should skip this record when dcat-distribution-strings is missing downloadURL", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"]
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/test.csv"
+                }
+            }
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
+    });
+    it("should use format from dcat-distribution-strings when dataset-format is missing", async () => {
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            autoDownloadFile: false
+        });
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": {},
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/test.csv"
+                }
+            }
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await expectNoThrowsAsync(() => onRecordFound(record, testEnv.registry));
+        expect(testEnv.createEmbeddingTextStub.firstCall.args[0]).to.deep.equal({
+            record,
+            format: "csv",
+            filePath: null,
+            url: "http://test.com/test.csv",
+            readonlyRegistry: testEnv.registry
+        });
+    });
+    it("should call deleteByQuery after indexing", async () => {
+        testEnv.createEmbeddingTextStub.resolves({
+            text: "embedding text: test1"
+        });
+        const userConfig = testEnv.updateUserConfig({
+            itemType: "storageObject",
+            formatTypes: ["csv"],
+            autoDownloadFile: false
+        });
+        testEnv.chunker.chunk
+            .withArgs("embedding text: test1")
+            .returns([{ text: "test1", length: 5, position: 0, overlap: 0 }]);
+        testEnv.embeddingApiClient.get
+            .withArgs(["test1"])
+            .resolves([[0.1, 0.2, 0.3]]);
+        const record = createRecord({
+            id: "id1",
+            aspects: {
+                "dataset-format": { format: "csv" },
+                "dcat-distribution-strings": {
+                    format: "csv",
+                    downloadURL: "http://test.com/file.csv"
+                }
+            }
+        });
+        const onRecordFound = onRecordFoundStorageObject(userConfig, testEnv.chunker, testEnv.embeddingApiClient, testEnv.opensearchApiClient, testEnv.minioClient, testEnv.registry);
+        await onRecordFound(record, testEnv.registry);
+        testEnv.expectSuccessCalls({
+            deleteByQueryCallCount: 1
+        });
+        testEnv.expectCalledWith(testEnv.opensearchApiClient.deleteByQuery, 0, {
+            index: userConfig.argv.semanticIndexerConfig.fullIndexName,
+            body: {
+                query: {
+                    bool: {
+                        filter: [
+                            { term: { indexerId: userConfig.id } },
+                            { term: { recordId: "id1" } },
+                            {
+                                range: {
+                                    createTime: {
+                                        lt: testEnv.getCurrentTimeString()
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                }
+            },
+            wait_for_completion: true,
+            conflicts: "proceed",
+            timeout: userConfig.timeout
+        });
+    });
+});
+//# sourceMappingURL=onRecordFoundStorageObject.spec.js.map

package/dist/test/onRecordFoundStorageObject.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"onRecordFoundStorageObject.spec.js","sourceRoot":"","sources":["../../src/test/onRecordFoundStorageObject.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAC9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IACxC,IAAI,OAAgC,CAAC;IAErC,UAAU,CAAC,GAAG,EAAE;QACZ,OAAO,GAAG,IAAI,uBAAuB,CAAC;YAClC,mBAAmB,EAAE,IAAI;SAC5B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACX,OAAO,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iGAAiG,EAAE,KAAK,IAAI,EAAE;QAC7G,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,EAAE;YACzD,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gGAAgG,EAAE,KAAK,IAAI,EAAE;QAC5G,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,IAAI,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAEjE,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,uBAAuB,CAAC,SAAS,CACrC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,YAAY,GAAG,QAAQ,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;QAC7C,CAAC,CACJ,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAExC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,WAAW,GAAG,OAAO,CAAC;QAC5B,MAAM,WAAW,GACb,wDAAwD,CAAC;QAC7D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC/C,OAAO,CAAC,WAAW,CAAC,YAAY;aAC3B,QAAQ,CAAC,WAAW,CAAC;aACrB,SAAS,CAAC,KAAK,IAAI,EAAE;YAClB,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;YAC5C,OAAO,YAAY,CAAC;QACxB,CAAC,CAAC,CAAC;QAEP,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,WAAW;iBAC3B;aACJ;SACJ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,uBAAuB,CAAC,SAAS,CACrC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,YAAY,GAAG,QAAQ,CAAC;YACxB,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC;QAC7C,CAAC,CACJ,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG;YACjB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAExC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QAClE,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;gBACpC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,2BAA2B;iBAC3C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;QACxF,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;YAC5B,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,OAAO,CAAC,uBAAuB;aAC1B,QAAQ,CAAC;YACN,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CAAC;aACD,QAAQ,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,aAAa,GAAG;YAClB;gBACI,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,KAAK;gBACf,cAAc,EAAE,OAAO,CAAC,wBAAwB;gBAChD,UAAU,EAAE,KAAK;gBACjB,gBAAgB,EAAE,OAAO;gBACzB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC1B,yBAAyB,EAAE,IAAI;gBAC/B,uBAAuB,EAAE,CAAC;gBAC1B,yBAAyB,EAAE,CAAC;gBAC5B,wBAAwB,EAAE,CAAC;gBAC3B,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE;aAC7C;SACJ,CAAC;QAEF,OAAO,CAAC,kBAAkB,CAAC;YACvB,4BAA4B,EAAE,CAAC;YAC/B,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,OAAO,CAAC,mBAAmB,CAAC,iBAAiB;aACxC,QAAQ,CAAC,MAAM,EAAE,IAAW,CAAC;aAC7B,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,mBAAmB,EAAE,KAAK;iBACrB,IAAI,EAAE;iBACN,OAAO,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE,EAAE;SACd,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE;gBACpB,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QAEF,MAAM,mBAAmB,CAAC,GAAG,EAAE,CAC3B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC1C,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CACnE;YACI,MAAM;YACN,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,IAAI;YACd,GAAG,EAAE,0BAA0B;YAC/B,gBAAgB,EAAE,OAAO,CAAC,QAAQ;SACrC,CACJ,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACtD,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC;YACrC,IAAI,EAAE,uBAAuB;SAChC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACxC,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,CAAC,KAAK,CAAC;YACpB,gBAAgB,EAAE,KAAK;SAC1B,CAAC,CAAC;QAEH,OAAO,CAAC,OAAO,CAAC,KAAK;aAChB,QAAQ,CAAC,uBAAuB,CAAC;aACjC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,kBAAkB,CAAC,GAAG;aACzB,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC;aACnB,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,YAAY,CAAC;YACxB,EAAE,EAAE,KAAK;YACT,OAAO,EAAE;gBACL,gBAAgB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;gBACnC,2BAA2B,EAAE;oBACzB,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,0BAA0B;iBAC1C;aACJ;SACJ,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,0BAA0B,CAC5C,UAAU,EACV,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE9C,OAAO,CAAC,kBAAkB,CAAC;YACvB,sBAAsB,EAAE,CAAC;SAC5B,CAAC,CAAC;QAEH,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,CAAC,EAAE;YACnE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,qBAAqB,CAAC,aAAa;YAC1D,IAAI,EAAE;gBACF,KAAK,EAAE;oBACH,IAAI,EAAE;wBACF,MAAM,EAAE;4BACJ,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE;4BACtC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;4BAC7B;gCACI,KAAK,EAAE;oCACH,UAAU,EAAE;wCACR,EAAE,EAAE,OAAO,CAAC,oBAAoB,EAAE;qCACrC;iCACJ;6BACJ;yBACJ;qBACJ;iBACJ;aACJ;YACD,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,UAAU,CAAC,OAAO;SAC9B,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@magda/semantic-indexer-framework",
+  "description": "MAGDA Semantic Indexer Framework",
+  "version": "5.2.0",
+  "type": "module",
+  "exports": {
+    ".": "./dist/index.js",
+    "./package.json": "./package.json",
+    "./dist/*.js": "./dist/*.js"
+  },
+  "engines": {
+    "node": ">=18.19.0"
+  },
+  "scripts": {
+    "prebuild": "rimraf dist tsconfig.tsbuildinfo",
+    "build": "yarn run compile",
+    "compile": "tsc -b && ts-module-alias-transformer dist",
+    "watch": "tsc -b --watch",
+    "test": "mocha",
+    "release": "npm publish || echo \"Skip releasing npm package @magda/semantic-indexer-framework.\""
+  },
+  "author": "",
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@magda/scripts": "^5.2.0",
+    "@types/chai": "^4.3.11",
+    "@types/mocha": "^10.0.6",
+    "@types/uuid": "^8.0.0",
+    "@types/yargs": "^12.0.8",
+    "chai": "^5.0.0-rc.0",
+    "mocha": "^10.2.0",
+    "typescript": "~5.3.3"
+  },
+  "dependencies": {
+    "@langchain/core": "^0.3.57",
+    "@langchain/textsplitters": "https://gitpkg.vercel.app/JHZhang2736/langchainjs/libs/langchain-textsplitters?textsplitters-build",
+    "@magda/esm-utils": "^1.0.1",
+    "@magda/minion-framework": "^5.2.0",
+    "@magda/typescript-common": "^5.2.0",
+    "@types/lodash": "^4.17.17",
+    "cross-fetch": "^4.0.0",
+    "minio": "^8.0.5",
+    "uuid": "^8.2.0",
+    "yargs": "^12.0.5"
+  },
+  "mocha": {
+    "import": "tsx/esm",
+    "spec": [
+      "src/test/**/*.spec.ts"
+    ]
+  },
+  "config": {
+    "docker": {
+      "include": "node_modules dist package.json"
+    }
+  },
+  "magda": {
+    "language": "typescript",
+    "categories": {
+      "minion": true,
+      "library": true,
+      "npmPackage": true
+    }
+  },
+  "_moduleMappings": {
+    "magda-typescript-common/src": "@magda/typescript-common/dist",
+    "magda-minion-framework/src": "@magda/minion-framework/dist"
+  },
+  "keywords": [
+    "Magda",
+    "SDK",
+    "Semantic",
+    "Indexer"
+  ],
+  "files": [
+    "dist"
+  ]
+}