@vegan-friendly/strapi-plugin-elasticsearch 0.2.2 → 0.2.4-alpha.0

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-    "version": "0.2.2",
+    "version": "0.2.4-alpha.0",
     "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
     "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
     "strapi": {
@@ -28,7 +28,9 @@
     "module": "dist/server.js",
     "types": "dist/index.d.ts",
     "files": [
-        "dist"
+        "dist",
+        "strapi-admin.js",
+        "strapi-server.js"
     ],
     "scripts": {
         "build": "tsc",

package/dist/server/index.d.ts

@@ -163,38 +163,7 @@ declare const _default: {
         };
         helper: ({ strapi }: {
             strapi: any;
-        }) => {
-            getElasticsearchInfo(): Promise<{
-                indexingCronSchedule: any;
-                elasticHost: any;
-                elasticUserName: any;
-                elasticCertificate: any;
-                elasticIndexAlias: any;
-                connected: any;
-                initialized: any;
-            }>;
-            isCollectionDraftPublish({ collectionName }: {
-                collectionName: any;
-            }): boolean;
-            getPopulateAttribute({ collectionName }: {
-                collectionName: any;
-            }): true | {
-                populate: never;
-            } | undefined;
-            getIndexItemId({ collectionName, itemId }: {
-                collectionName: any;
-                itemId: any;
-            }): string;
-            getCurrentIndexName(indexPrefix?: string): Promise<string>;
-            getIncrementedIndexName(indexPrefix?: string): Promise<string>;
-            storeCurrentIndexName(indexName: any, indexPrefix?: string): Promise<void>;
-            modifySubfieldsConfigForExtractor(collectionConfig: any): any;
-            extractDataToIndex({ collectionName, data, collectionConfig }: {
-                collectionName: any;
-                data: any;
-                collectionConfig: any;
-            }): {};
-        };
+        }) => import("./types/helper-service.type").HelperService;
         transformContent: {
             transform({ content, from }: {
                 content: any;

package/dist/server/services/es-interface.js

@@ -158,4 +158,11 @@ exports.default = ({ strapi }) => ({
             throw err;
         }
     },
+    async listIndicesByPattern(pattern = 'restaurants*') {
+        const results = await client.cat.indices({
+            index: pattern,
+            format: 'json',
+        });
+        return results.map((index) => index.index).filter((index) => index != null);
+    },
 });

package/dist/server/services/helper.d.ts

@@ -1,35 +1,5 @@
+import { HelperService } from '../types/helper-service.type';
 declare const _default: ({ strapi }: {
     strapi: any;
-}) => {
-    getElasticsearchInfo(): Promise<{
-        indexingCronSchedule: any;
-        elasticHost: any;
-        elasticUserName: any;
-        elasticCertificate: any;
-        elasticIndexAlias: any;
-        connected: any;
-        initialized: any;
-    }>;
-    isCollectionDraftPublish({ collectionName }: {
-        collectionName: any;
-    }): boolean;
-    getPopulateAttribute({ collectionName }: {
-        collectionName: any;
-    }): true | {
-        populate: never;
-    } | undefined;
-    getIndexItemId({ collectionName, itemId }: {
-        collectionName: any;
-        itemId: any;
-    }): string;
-    getCurrentIndexName(indexPrefix?: string): Promise<string>;
-    getIncrementedIndexName(indexPrefix?: string): Promise<string>;
-    storeCurrentIndexName(indexName: any, indexPrefix?: string): Promise<void>;
-    modifySubfieldsConfigForExtractor(collectionConfig: any): any;
-    extractDataToIndex({ collectionName, data, collectionConfig }: {
-        collectionName: any;
-        data: any;
-        collectionConfig: any;
-    }): {};
-};
+}) => HelperService;
 export default _default;

package/dist/server/services/helper.js

@@ -7,13 +7,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fp_1 = require("lodash/fp");
 const transform_content_1 = __importDefault(require("./transform-content"));
 const defaultIndexPrefix = 'strapi-plugin-elasticsearch-index';
-const getPluginStore = () => {
-    return strapi.store({
-        environment: '',
-        type: 'plugin',
-        name: 'elasticsearch',
-    });
-};
 const getModelPopulationAttributes = (model) => {
     if (model.uid === 'plugin::upload.file') {
         const { related, ...attributes } = model.attributes;
@@ -165,115 +158,117 @@ function extractSubfieldData({ config, data }) {
     }
     return returnData;
 }
-exports.default = ({ strapi }) => ({
-    async getElasticsearchInfo() {
-        const configureService = strapi.plugins['elasticsearch'].services.configureIndexing;
-        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
-        const pluginConfig = await strapi.config.get('plugin.elasticsearch');
-        const connected = pluginConfig.searchConnector && pluginConfig.searchConnector.host ? await esInterface.checkESConnection() : false;
-        return {
-            indexingCronSchedule: pluginConfig.indexingCronSchedule || 'Not configured',
-            elasticHost: pluginConfig.searchConnector ? pluginConfig.searchConnector.host || 'Not configured' : 'Not configured',
-            elasticUserName: pluginConfig.searchConnector ? pluginConfig.searchConnector.username || 'Not configured' : 'Not configured',
-            elasticCertificate: pluginConfig.searchConnector ? pluginConfig.searchConnector.certificate || 'Not configured' : 'Not configured',
-            elasticIndexAlias: pluginConfig.indexAliasName || 'Not configured',
-            connected: connected,
-            initialized: configureService.isInitialized(),
-        };
-    },
-    isCollectionDraftPublish({ collectionName }) {
-        const model = strapi.getModel(collectionName);
-        return model.attributes.publishedAt ? true : false;
-    },
-    getPopulateAttribute({ collectionName }) {
-        //TODO : We currently have set populate to upto 4 levels, should
-        //this be configurable or a different default value?
-        return getFullPopulateObject(collectionName, 4, []);
-    },
-    getIndexItemId({ collectionName, itemId }) {
-        return collectionName + '::' + itemId;
-    },
-    async getCurrentIndexName(indexPrefix = defaultIndexPrefix) {
-        const pluginStore = getPluginStore();
-        const settings = (await pluginStore.get({ key: 'configsettings' }));
-        let indexName = indexPrefix + '_000001';
-        if (settings) {
-            const objSettings = JSON.parse(settings);
-            if (Object.keys(objSettings).includes('indexConfig')) {
-                const idxConfig = objSettings['indexConfig'];
-                if (idxConfig[indexPrefix]) {
-                    indexName = idxConfig[indexPrefix];
-                }
-            }
-        }
-        return indexName;
-    },
-    async getIncrementedIndexName(indexPrefix = defaultIndexPrefix) {
-        const currentIndexName = await this.getCurrentIndexName(indexPrefix);
-        const number = parseInt(currentIndexName.split(indexPrefix + '_')[1]);
-        return indexPrefix + '_' + String(number + 1).padStart(6, '0');
-    },
-    async storeCurrentIndexName(indexName, indexPrefix = defaultIndexPrefix) {
-        const pluginStore = getPluginStore();
-        const settings = (await pluginStore.get({ key: 'configsettings' }));
-        if (settings) {
-            const objSettings = JSON.parse(settings);
-            objSettings['indexConfig'] = {
-                ...objSettings['indexConfig'],
-                [indexPrefix]: indexName,
+exports.default = ({ strapi }) => {
+    const getEsInterface = () => strapi.plugins['elasticsearch'].services.esInterface;
+    return {
+        async getElasticsearchInfo() {
+            const configureService = strapi.plugins['elasticsearch'].services.configureIndexing;
+            const pluginConfig = await strapi.config.get('plugin.elasticsearch');
+            const connected = pluginConfig.searchConnector && pluginConfig.searchConnector.host ? await getEsInterface().checkESConnection() : false;
+            return {
+                indexingCronSchedule: pluginConfig.indexingCronSchedule || 'Not configured',
+                elasticHost: pluginConfig.searchConnector ? pluginConfig.searchConnector.host || 'Not configured' : 'Not configured',
+                elasticUserName: pluginConfig.searchConnector ? pluginConfig.searchConnector.username || 'Not configured' : 'Not configured',
+                elasticCertificate: pluginConfig.searchConnector ? pluginConfig.searchConnector.certificate || 'Not configured' : 'Not configured',
+                elasticIndexAlias: pluginConfig.indexAliasName || 'Not configured',
+                connected: connected,
+                initialized: configureService.isInitialized(),
             };
-            await pluginStore.set({ key: 'configsettings', value: JSON.stringify(objSettings) });
-        }
-        else {
-            const newSettings = JSON.stringify({ indexConfig: { [indexPrefix]: indexName } });
-            await pluginStore.set({ key: 'configsettings', value: newSettings });
-        }
-    },
-    modifySubfieldsConfigForExtractor(collectionConfig) {
-        const collectionName = Object.keys(collectionConfig)[0];
-        const attributes = Object.keys(collectionConfig[collectionName]);
-        for (let r = 0; r < attributes.length; r++) {
-            const attr = attributes[r];
-            const attribFields = Object.keys(collectionConfig[collectionName][attr]);
-            if (attribFields.includes('subfields')) {
-                const subfielddata = collectionConfig[collectionName][attr]['subfields'];
-                if (subfielddata.length > 0) {
-                    try {
-                        const subfieldjson = JSON.parse(subfielddata);
-                        if (Object.keys(subfieldjson).includes('subfields'))
-                            collectionConfig[collectionName][attr]['subfields'] = subfieldjson['subfields'];
-                    }
-                    catch (err) {
-                        continue;
+        },
+        isCollectionDraftPublish({ collectionName }) {
+            const model = strapi.getModel(collectionName);
+            return model.attributes.publishedAt ? true : false;
+        },
+        getPopulateAttribute({ collectionName }) {
+            //TODO : We currently have set populate to upto 4 levels, should
+            //this be configurable or a different default value?
+            return getFullPopulateObject(collectionName, 4, []);
+        },
+        getIndexItemId({ collectionName, itemId }) {
+            return collectionName + '::' + itemId;
+        },
+        async getCurrentIndexName(indexPrefix = defaultIndexPrefix) {
+            const indices = await getEsInterface().listIndicesByPattern(indexPrefix + '_*');
+            if (indices.length === 0) {
+                return indexPrefix + '_000000';
+            }
+            const sortedIndices = indices.sort(sortByIndexNumber(indexPrefix)).reverse();
+            return sortedIndices[0];
+        },
+        async getIncrementedIndexName(indexPrefix = defaultIndexPrefix) {
+            const currentIndexName = await this.getCurrentIndexName(indexPrefix);
+            const number = parseInt(currentIndexName.split(indexPrefix + '_')[1]);
+            return indexPrefix + '_' + String(number + 1).padStart(6, '0');
+        },
+        async deleteOldIndices(indexPrefix = defaultIndexPrefix) {
+            const esInterface = getEsInterface();
+            const indices = await esInterface.listIndicesByPattern(indexPrefix + '_*');
+            const sortedIndices = indices.sort(sortByIndexNumber(indexPrefix)).reverse();
+            const indicesToDelete = sortedIndices.slice(1);
+            for (let index of indicesToDelete) {
+                await esInterface.deleteIndex(index);
+            }
+            return indicesToDelete;
+        },
+        modifySubfieldsConfigForExtractor(collectionConfig) {
+            const collectionName = Object.keys(collectionConfig)[0];
+            const attributes = Object.keys(collectionConfig[collectionName]);
+            for (let r = 0; r < attributes.length; r++) {
+                const attr = attributes[r];
+                const attribFields = Object.keys(collectionConfig[collectionName][attr]);
+                if (attribFields.includes('subfields')) {
+                    const subfielddata = collectionConfig[collectionName][attr]['subfields'];
+                    if (subfielddata.length > 0) {
+                        try {
+                            const subfieldjson = JSON.parse(subfielddata);
+                            if (Object.keys(subfieldjson).includes('subfields'))
+                                collectionConfig[collectionName][attr]['subfields'] = subfieldjson['subfields'];
+                        }
+                        catch (err) {
+                            continue;
+                        }
                     }
                 }
             }
-        }
-        return collectionConfig;
-    },
-    extractDataToIndex({ collectionName, data, collectionConfig }) {
-        collectionConfig = this.modifySubfieldsConfigForExtractor(collectionConfig);
-        const fti = Object.keys(collectionConfig[collectionName]);
-        const document = {};
-        for (let k = 0; k < fti.length; k++) {
-            const fieldConfig = collectionConfig[collectionName][fti[k]];
-            if (fieldConfig.index) {
-                let val = null;
-                if (Object.keys(fieldConfig).includes('subfields')) {
-                    val = extractSubfieldData({ config: fieldConfig['subfields'], data: data[fti[k]] });
-                    val = val ? val.trim() : val;
-                }
-                else {
-                    val = data[fti[k]];
-                    if (Object.keys(fieldConfig).includes('transform') && fieldConfig['transform'] === 'markdown')
-                        val = transform_content_1.default.transform({ content: val, from: 'markdown' });
+            return collectionConfig;
+        },
+        extractDataToIndex({ collectionName, data, collectionConfig }) {
+            collectionConfig = this.modifySubfieldsConfigForExtractor(collectionConfig);
+            const fti = Object.keys(collectionConfig[collectionName]);
+            const document = {};
+            for (let k = 0; k < fti.length; k++) {
+                const fieldConfig = collectionConfig[collectionName][fti[k]];
+                if (fieldConfig.index) {
+                    let val = null;
+                    if (Object.keys(fieldConfig).includes('subfields')) {
+                        val = extractSubfieldData({ config: fieldConfig['subfields'], data: data[fti[k]] });
+                        val = val ? val.trim() : val;
+                    }
+                    else {
+                        val = data[fti[k]];
+                        if (Object.keys(fieldConfig).includes('transform') && fieldConfig['transform'] === 'markdown')
+                            val = transform_content_1.default.transform({ content: val, from: 'markdown' });
+                    }
+                    if (Object.keys(fieldConfig).includes('searchFieldName'))
+                        document[fieldConfig['searchFieldName']] = val;
+                    else
+                        document[fti[k]] = val;
                 }
-                if (Object.keys(fieldConfig).includes('searchFieldName'))
-                    document[fieldConfig['searchFieldName']] = val;
-                else
-                    document[fti[k]] = val;
             }
-        }
-        return document;
-    },
-});
+            return document;
+        },
+    };
+};
+/**
+ * sort index names by index number, e.g:
+ * restaurants_000001, restaurants_000002, restaurants_000003
+ * @param indexPrefix index alias, usually
+ * @returns
+ */
+function sortByIndexNumber(indexPrefix) {
+    return (a, b) => {
+        const numA = parseInt(a.split(indexPrefix + '_')[1]);
+        const numB = parseInt(b.split(indexPrefix + '_')[1]);
+        return numA - numB;
+    };
+}

package/dist/server/services/index.d.ts

@@ -59,38 +59,7 @@ declare const _default: {
     };
     helper: ({ strapi }: {
         strapi: any;
-    }) => {
-        getElasticsearchInfo(): Promise<{
-            indexingCronSchedule: any;
-            elasticHost: any;
-            elasticUserName: any;
-            elasticCertificate: any;
-            elasticIndexAlias: any;
-            connected: any;
-            initialized: any;
-        }>;
-        isCollectionDraftPublish({ collectionName }: {
-            collectionName: any;
-        }): boolean;
-        getPopulateAttribute({ collectionName }: {
-            collectionName: any;
-        }): true | {
-            populate: never;
-        } | undefined;
-        getIndexItemId({ collectionName, itemId }: {
-            collectionName: any;
-            itemId: any;
-        }): string;
-        getCurrentIndexName(indexPrefix?: string): Promise<string>;
-        getIncrementedIndexName(indexPrefix?: string): Promise<string>;
-        storeCurrentIndexName(indexName: any, indexPrefix?: string): Promise<void>;
-        modifySubfieldsConfigForExtractor(collectionConfig: any): any;
-        extractDataToIndex({ collectionName, data, collectionConfig }: {
-            collectionName: any;
-            data: any;
-            collectionConfig: any;
-        }): {};
-    };
+    }) => import("../types/helper-service.type").HelperService;
     transformContent: {
         transform({ content, from }: {
             content: any;

package/dist/server/services/perform-indexing.js

@@ -31,11 +31,9 @@ exports.default = ({ strapi }) => ({
                 //Step 4 : Move the alias to this new index
                 await esInterface.attachAliasToIndex(newIndexName);
                 console.log('strapi-plugin-elasticsearch : Attaching the newly created index to the alias.');
-                //Step 3 : Update the search-indexing-name
-                await helper.storeCurrentIndexName(newIndexName);
-                console.log('strapi-plugin-elasticsearch : Deleting the previous index : ', oldIndexName);
+                console.log('strapi-plugin-elasticsearch : Deleting the previous indices');
                 //Step 5 : Delete the previous index
-                await esInterface.deleteIndex(oldIndexName);
+                await helper.deleteOldIndices();
                 await logIndexingService.recordIndexingPass('Request to immediately re-index site-wide content completed successfully.');
                 return true;
             }

package/dist/server/services/virtual-collections-indexer.js

@@ -30,8 +30,8 @@ exports.default = ({ strapi }) => {
                 const itemData = results[0];
                 const esInterface = getElasticsearchService();
                 const helper = getHelperService();
-                const indexItemId = helper.getIndexItemId(collectionName, itemId);
-                const indexName = await helper.getCurrentIndexName();
+                const indexItemId = helper.getIndexItemId({ collectionName, itemId });
+                const indexName = await helper.getCurrentIndexName(collection.indexAlias);
                 await esInterface.indexDataToSpecificIndex({ itemId: indexItemId, itemData }, indexName);
                 strapi.log.debug(`Indexed virtual item: ${collectionName}:${itemId}`);
                 return itemData;
@@ -57,14 +57,19 @@ exports.default = ({ strapi }) => {
          */
         async reindex(collection) {
             const collectionName = collection.collectionName;
-            const indexAlias = collection.indexAlias;
+            const privateIndexAlias = collection.indexAlias;
             const helper = getHelperService();
-            const oldIndexName = await helper.getCurrentIndexName(indexAlias);
-            const newIndexName = await helper.getIncrementedIndexName(indexAlias);
             let timestamp = Date.now();
             try {
                 const esInterface = getElasticsearchService();
-                await esInterface.createIndex(newIndexName, collection.mappings);
+                let indexName;
+                if (privateIndexAlias) {
+                    indexName = await helper.getIncrementedIndexName(privateIndexAlias);
+                    await esInterface.createIndex(indexName, collection.mappings);
+                }
+                else {
+                    indexName = await helper.getCurrentIndexName();
+                }
                 let page = 0;
                 let hasMoreData = true;
                 let totalIndexed = 0;
@@ -80,19 +85,20 @@ exports.default = ({ strapi }) => {
                         operations.push({ itemId, itemData });
                     }
                     if (operations.length > 0) {
-                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, newIndexName)));
+                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName)));
                     }
                     totalIndexed += pageData.length;
                     page++;
                 }
                 strapi.log.info(`Reindexed ${totalIndexed} items for virtual collection: ${collectionName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}. now updating alias.`);
-                timestamp = Date.now();
-                await esInterface.attachAliasToIndex(newIndexName, indexAlias, collection.mappings);
-                strapi.log.info(`Done attachAliasToIndex alias ${indexAlias} to index ${newIndexName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
-                timestamp = Date.now();
-                await helper.storeCurrentIndexName(newIndexName, indexAlias);
-                await esInterface.deleteIndex(oldIndexName);
-                strapi.log.info(`Done deleting ${oldIndexName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                if (privateIndexAlias) {
+                    timestamp = Date.now();
+                    await esInterface.attachAliasToIndex(indexName, privateIndexAlias, collection.mappings);
+                    strapi.log.info(`Done attachAliasToIndex alias ${privateIndexAlias} to index ${indexName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                    timestamp = Date.now();
+                    const oldIndicesDeleted = await helper.deleteOldIndices(privateIndexAlias);
+                    strapi.log.info(`Done deleting ${oldIndicesDeleted.length} old indices: ${oldIndicesDeleted}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                }
                 return totalIndexed;
             }
             catch (error) {

package/dist/server/types/esInterface.type.d.ts

@@ -69,4 +69,5 @@ export interface EsInterfaceService {
      * @returns A promise that resolves with the search results.
      */
     searchData(searchQuery: any): Promise<any>;
+    listIndicesByPattern(pattern: string): Promise<string[]>;
 }

package/dist/server/types/helper-service.type.d.ts

@@ -0,0 +1,33 @@
+export type ElasticsearchInfo = {
+    indexingCronSchedule: string;
+    elasticHost: string;
+    elasticUserName: string;
+    elasticCertificate: string;
+    elasticIndexAlias: number;
+    connected: string;
+    initialized: boolean;
+};
+export interface HelperService {
+    getElasticsearchInfo(): Promise<ElasticsearchInfo>;
+    isCollectionDraftPublish(args: {
+        collectionName: string;
+    }): boolean;
+    getPopulateAttribute(args: {
+        collectionName: string;
+    }): true | {
+        populate: object;
+    } | undefined;
+    getIndexItemId(args: {
+        collectionName: string;
+        itemId: number;
+    }): string;
+    getCurrentIndexName(indexAlias?: string): Promise<string>;
+    getIncrementedIndexName(indexPrefix: string): Promise<string>;
+    deleteOldIndices(indexAlias?: string): Promise<string[]>;
+    modifySubfieldsConfigForExtractor(collectionConfig: object): object;
+    extractDataToIndex(args: {
+        collectionName: string;
+        data: object;
+        collectionConfig: object;
+    }): any;
+}

package/dist/server/types/helper-service.type.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+let a = {
+    a: 1,
+};
+const example = { foo: "bar" };
+a = example;

package/dist/server/types/virtual-collections.type.d.ts

@@ -1,12 +1,16 @@
 import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 export type VirtualCollectionConfig<T extends StrapiEntity> = {
     /**
+     * Optional -
      * The alias of the latest index in Elasticsearch.
      * It also serves as a prefix to actual indexes created in Elasticsearch.
      * The actuall index name will be the `${indexNameBase}_${ind}`, e.g `restaurants_000001` and so on.
      * and the alias will be the `indexNameBase`.
+     *
+     * If you don't provide this, the default index name will be used as the alias.
+     * Omit this property if you want to use the default index for all collections.
      */
-    indexAlias: string;
+    indexAlias?: string;
     collectionName: string;
     extractData: (page: number, pageSize?: number) => Promise<T[]>;
     extractById: (ids: number[]) => Promise<T[]>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.2.2",
+  "version": "0.2.4-alpha.0",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {
@@ -28,7 +28,9 @@
   "module": "dist/server.js",
   "types": "dist/index.d.ts",
   "files": [
-    "dist"
+    "dist",
+    "strapi-admin.js",
+    "strapi-server.js"
   ],
   "scripts": {
     "build": "tsc",
@@ -61,4 +63,4 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.8.3"
   }
-}
+}

package/strapi-admin.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('./admin/src').default;

package/strapi-server.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('./dist/server');