@vegan-friendly/strapi-plugin-elasticsearch 0.0.8

package/server/services/helper.js

@@ -0,0 +1,305 @@
+
+///START : via https://raw.githubusercontent.com/Barelydead/strapi-plugin-populate-deep/main/server/helpers/index.js
+
+const { isEmpty, merge } = require("lodash/fp");
+const transformServiceProvider = require('./transform-content');
+
+const getPluginStore = () => {
+    return strapi.store({
+      environment: '',
+      type: 'plugin',
+      name: 'elasticsearch',
+    });
+  }
+
+
+const getModelPopulationAttributes = (model) => {
+  if (model.uid === "plugin::upload.file") {
+    const { related, ...attributes } = model.attributes;
+    return attributes;
+  }
+
+  return model.attributes;
+};
+
+const getFullPopulateObject = (modelUid, maxDepth = 20, ignore) => {
+  const skipCreatorFields = true;
+
+  if (maxDepth <= 1) {
+    return true;
+  }
+  if (modelUid === "admin::user" && skipCreatorFields) {
+    return undefined;
+  }
+
+  const populate = {};
+  const model = strapi.getModel(modelUid);
+  if (ignore && !ignore.includes(model.collectionName)) ignore.push(model.collectionName)
+  for (const [key, value] of Object.entries(
+    getModelPopulationAttributes(model)
+  )) {
+    if (ignore?.includes(key)) continue
+    if (value) {
+      if (value.type === "component") {
+        populate[key] = getFullPopulateObject(value.component, maxDepth - 1);
+      } else if (value.type === "dynamiczone") {
+        const dynamicPopulate = value.components.reduce((prev, cur) => {
+          const curPopulate = getFullPopulateObject(cur, maxDepth - 1);
+          return curPopulate === true ? prev : merge(prev, curPopulate);
+        }, {});
+        populate[key] = isEmpty(dynamicPopulate) ? true : dynamicPopulate;
+      } else if (value.type === "relation") {
+        const relationPopulate = getFullPopulateObject(
+          value.target,
+          (key === 'localizations') && maxDepth > 2 ? 1 : maxDepth - 1,
+          ignore
+        );
+        if (relationPopulate) {
+          populate[key] = relationPopulate;
+        }
+      } else if (value.type === "media") {
+        populate[key] = true;
+      }
+    }
+  }
+  return isEmpty(populate) ? true : { populate };
+};
+
+///END : via https://raw.githubusercontent.com/Barelydead/strapi-plugin-populate-deep/main/server/helpers/index.js
+  
+
+/*
+//Example config to cover extraction cases
+            collectionConfig[collectionName] = {
+                'major' : {index: true},
+                'sections' : { index: true, searchFieldName: 'information',
+                    'subfields' : [
+                        { 'component' : 'try.paragraph',
+                            'field' : 'Text'}, 
+                        { 'component' : 'try.paragraph',
+                            'field' : 'Heading'},
+                        { 'component' : 'try.footer',
+                            'field' : 'footer_link',
+                            'subfields' :[ {
+                                'component' : 'try.link',
+                                'field' : 'display_text'
+                            }] 
+                        }] },
+                'seo_details' : {
+                    index: true, searchFieldName: 'seo',
+                    'subfields' : [
+                        {
+                            'component' : 'try.seo',
+                            'field' : 'meta_description'
+                        }
+                    ]
+                },
+                'changelog' : {
+                    index: true, searchFieldName: 'breakdown',
+                    'subfields' : [
+                        {
+                            'component' : 'try.revision',
+                            'field' : 'summary'
+                        }
+                    ]
+                }
+            }
+*/            
+function extractSubfieldData({config, data }) {
+    let returnData = '';
+    if (Array.isArray(data))
+    {
+        const dynDataItems = data;
+        for (let r=0; r< dynDataItems.length; r++)
+        {
+            const extractItem = dynDataItems[r];
+            for (let s=0; s<config.length; s++)
+            {
+                const conf = config[s];
+                if (Object.keys(extractItem).includes('__component'))
+                {
+                    if (conf.component === extractItem.__component &&
+                        !Object.keys(conf).includes('subfields') &&
+                        typeof extractItem[conf['field']] !== "undefined" &&
+                        extractItem[conf['field']])
+                    {
+                        let val = extractItem[conf['field']]
+                        if (Object.keys(conf).includes('transform')
+                            && conf['transform'] === 'markdown')
+                            val = transformServiceProvider.transform({content: val, from: 'markdown'});
+                        returnData = returnData + '\n' + val;
+                    }
+                    else if (conf.component === extractItem.__component &&
+                        Object.keys(conf).includes('subfields'))
+                    {
+                        returnData = returnData + '\n' + extractSubfieldData({
+                            config: conf['subfields'], data: extractItem[conf['field']]}); 
+                    }
+                }
+                else
+                {
+                    if (!Object.keys(conf).includes('subfields') &&
+                    typeof extractItem[conf['field']] !== "undefined" &&
+                    extractItem[conf['field']])
+                    {
+                        let val = extractItem[conf['field']]
+                        if (Object.keys(conf).includes('transform')
+                            && conf['transform'] === 'markdown')
+                            val = transformServiceProvider.transform({content: val, from: 'markdown'});
+                        returnData = returnData + '\n' + val;
+                    }
+                    else if (Object.keys(conf).includes('subfields'))
+                    {
+                        returnData = returnData + '\n' + extractSubfieldData({
+                            config: conf['subfields'], data: extractItem[conf['field']]}); 
+                    }
+                }
+            }
+        }
+    }
+    else //for single component as a field
+    {
+        for (let s=0; s<config.length; s++)
+        {
+            const conf = config[s];
+            if (!Object.keys(conf).includes('subfields') &&
+            typeof data[conf['field']] !== "undefined" &&
+            data[conf['field']])
+                returnData = returnData + '\n' + data[conf['field']]
+            else if (Object.keys(conf).includes('subfields'))
+            {
+                returnData = returnData + '\n' + extractSubfieldData({
+                    config: conf['subfields'], data: data[conf['field']]}); 
+            }
+        }
+    }
+    return returnData;    
+}
+
+module.exports = ({ strapi }) => ({
+    async getElasticsearchInfo() {
+        const configureService = strapi.plugins['elasticsearch'].services.configureIndexing;
+        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
+        const pluginConfig = await strapi.config.get('plugin.elasticsearch');
+      
+        const connected = pluginConfig.searchConnector && pluginConfig.searchConnector.host
+         ? await esInterface.checkESConnection() : false;
+
+        return {
+            indexingCronSchedule : pluginConfig.indexingCronSchedule || "Not configured",
+            elasticHost : pluginConfig.searchConnector ? 
+                            pluginConfig.searchConnector.host || "Not configured" : "Not configured",
+            elasticUserName : pluginConfig.searchConnector ? 
+                            pluginConfig.searchConnector.username || "Not configured" : "Not configured",
+            elasticCertificate : pluginConfig.searchConnector ? 
+            pluginConfig.searchConnector.certificate || "Not configured" : "Not configured",
+            elasticIndexAlias : pluginConfig.indexAliasName || "Not configured",
+            connected : connected,
+            initialized : configureService.isInitialized()
+        }
+    },
+    isCollectionDraftPublish({collectionName}) {
+        const model = strapi.getModel(collectionName);
+        return model.attributes.publishedAt ? true : false
+    },
+    getPopulateAttribute({collectionName}) {
+        //TODO : We currently have set populate to upto 4 levels, should
+        //this be configurable or a different default value?
+        return getFullPopulateObject(collectionName, 4, []);
+    },
+    getIndexItemId ({collectionName, itemId}) {
+        return collectionName+'::' + itemId;
+    },
+    async getCurrentIndexName () {
+        const pluginStore = getPluginStore();
+        const settings = await pluginStore.get({ key: 'configsettings' });
+        let indexName = 'strapi-plugin-elasticsearch-index_000001';
+        if (settings)
+        {
+          const objSettings = JSON.parse(settings);
+          if (Object.keys(objSettings).includes('indexConfig'))
+          {
+            const idxConfig = objSettings['indexConfig'];
+            indexName = idxConfig['name'];
+          }
+        }
+        return indexName;
+    },
+    async getIncrementedIndexName () {
+        const currentIndexName = await this.getCurrentIndexName();
+        const number = parseInt(currentIndexName.split('index_')[1]);
+        return 'strapi-plugin-elasticsearch-index_' + String(number+1).padStart(6,'0');
+    },
+    async storeCurrentIndexName (indexName) {
+        const pluginStore = getPluginStore();
+        const settings = await pluginStore.get({ key: 'configsettings' });
+        if (settings)
+        {
+            const objSettings = JSON.parse(settings);
+            objSettings['indexConfig'] = {'name' : indexName};
+            await pluginStore.set({ key: 'configsettings', value : JSON.stringify(objSettings)});      
+        }
+        else
+        {
+            const newSettings =  JSON.stringify({'indexConfig' : {'name' : indexName}})
+            await pluginStore.set({ key: 'configsettings', value : newSettings});      
+        }
+    },
+    modifySubfieldsConfigForExtractor(collectionConfig) {
+        const collectionName = Object.keys(collectionConfig)[0];
+        const attributes = Object.keys(collectionConfig[collectionName]);
+        for (let r=0; r< attributes.length; r++)
+        {
+            const attr = attributes[r];
+            const attribFields = Object.keys(collectionConfig[collectionName][attr]);
+            if (attribFields.includes('subfields'))
+            {
+                const subfielddata = collectionConfig[collectionName][attr]['subfields'];
+                if (subfielddata.length > 0)
+                {
+                    try {
+                        const subfieldjson = JSON.parse(subfielddata)
+                        if (Object.keys(subfieldjson).includes('subfields'))
+                            collectionConfig[collectionName][attr]['subfields'] = subfieldjson['subfields']
+                    }
+                    catch(err)
+                    {
+                        continue;
+                    }
+                }
+            }
+        }
+        return collectionConfig;
+    },
+    extractDataToIndex({collectionName, data, collectionConfig}) {
+        collectionConfig = this.modifySubfieldsConfigForExtractor(collectionConfig);
+        const fti = Object.keys(collectionConfig[collectionName]);
+        const document = {}
+        for (let k = 0; k < fti.length; k++)
+        {
+            const fieldConfig = collectionConfig[collectionName][fti[k]];
+            if (fieldConfig.index)
+            {
+                let val = null;
+                if (Object.keys(fieldConfig).includes('subfields'))
+                {
+                    val = extractSubfieldData({config: fieldConfig['subfields'], data: data[fti[k]]})
+                    val = val ? val.trim() : val
+                }
+                else
+                {
+                    val = data[fti[k]];
+                    if (Object.keys(fieldConfig).includes('transform') && 
+                        fieldConfig['transform'] === 'markdown')
+                        val = transformServiceProvider.transform({content: val, from: 'markdown'});
+                }
+                    
+                if (Object.keys(fieldConfig).includes('searchFieldName'))
+                    document[fieldConfig['searchFieldName']] = val;
+                else
+                    document[fti[k]] = val;
+            }
+        }
+        return document;
+    }    
+});

package/server/services/index.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const configureIndexing = require('./configure-indexing');
+const scheduleIndexing = require('./schedule-indexing');
+const esInterface = require('./es-interface');
+const indexer = require('./perform-indexing');
+const logIndexing = require('./log-indexing');
+const helper = require('./helper');
+const transformContent = require('./transform-content');
+
+module.exports = {
+  configureIndexing,
+  scheduleIndexing,
+  esInterface,
+  indexer,
+  logIndexing,
+  helper,
+  transformContent
+};

package/server/services/log-indexing.js

@@ -0,0 +1,26 @@
+module.exports = ({ strapi }) => ({
+    async recordIndexingPass(message) {
+        const entry = await strapi.entityService.create('plugin::elasticsearch.indexing-log', {
+            data : {
+                status: 'pass',
+                details: message
+            }
+        });
+    },
+    async recordIndexingFail(message) {
+        const entry = await strapi.entityService.create('plugin::elasticsearch.indexing-log', {
+            data : {
+                status: 'fail',
+                details: String(message)
+            }
+        });
+    },
+    async fetchIndexingLogs(count = 50) {
+        const records = await strapi.entityService.findMany('plugin::elasticsearch.indexing-log', {
+            sort: { createdAt: 'DESC' },
+            start: 0,
+            limit: count
+        });
+        return records;
+    }
+});

package/server/services/perform-indexing.js

@@ -0,0 +1,173 @@
+
+module.exports = ({ strapi }) => ({
+
+    async rebuildIndex() {
+        const helper = strapi.plugins['elasticsearch'].services.helper;
+        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
+        const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
+        const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
+        const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
+
+        try
+        {
+            console.log('strapi-plugin-elasticsearch : Request to rebuild the index received.')
+            const oldIndexName = await helper.getCurrentIndexName();
+            console.log('strapi-plugin-elasticsearch : Recording the previous index name : ', oldIndexName);
+    
+            //Step 1 : Create a new index
+            const newIndexName = await helper.getIncrementedIndexName();
+            await esInterface.createIndex(newIndexName);
+            console.log('strapi-plugin-elasticsearch : Created new index with name : ', newIndexName);
+            
+    
+            //Step 2 : Index all the stuff on this new index
+            console.log('strapi-plugin-elasticsearch : Starting to index all data into the new index.');
+            const item = await scheduleIndexingService.addFullSiteIndexingTask();
+            
+            if (item.id)
+            {
+                const cols = await configureIndexingService.getCollectionsConfiguredForIndexing();
+                for (let r=0; r< cols.length; r++)
+                    await this.indexCollection(cols[r], newIndexName);
+                    
+                await scheduleIndexingService.markIndexingTaskComplete(item.id);    
+    
+                console.log('strapi-plugin-elasticsearch : Indexing of data into the new index complete.');
+                //Step 4 : Move the alias to this new index
+                await esInterface.attachAliasToIndex(newIndexName);;
+                console.log('strapi-plugin-elasticsearch : Attaching the newly created index to the alias.')
+                //Step 3 : Update the search-indexing-name
+                await helper.storeCurrentIndexName(newIndexName);
+    
+                console.log('strapi-plugin-elasticsearch : Deleting the previous index : ', oldIndexName);
+                //Step 5 : Delete the previous index
+                await esInterface.deleteIndex(oldIndexName)
+                await logIndexingService.recordIndexingPass('Request to immediately re-index site-wide content completed successfully.');
+
+                return true;
+            }
+            else
+            {
+                await logIndexingService.recordIndexingFail('An error was encountered while trying site-wide re-indexing of content.');
+                return false;
+            }      
+        }
+        catch(err)
+        {
+            console.log('strapi-plugin-elasticsearch : searchController : An error was encountered while re-indexing.')
+            console.log(err);
+            await logIndexingService.recordIndexingFail(err);
+        }
+    },
+    async indexCollection(collectionName, indexName = null) {
+        const helper = strapi.plugins['elasticsearch'].services.helper;
+        const populateAttrib = helper.getPopulateAttribute({collectionName});
+        const isCollectionDraftPublish = helper.isCollectionDraftPublish({collectionName});
+        const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
+        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
+        if (indexName === null)
+            indexName = await helper.getCurrentIndexName();
+        let entries = [];
+        if (isCollectionDraftPublish)
+        {
+            entries = await strapi.entityService.findMany(collectionName, {
+                sort: { createdAt: 'DESC' },
+                populate: populateAttrib['populate'],
+                filters: {
+                    publishedAt: {
+                        $notNull: true
+                    }
+                }
+            });
+        }
+        else
+        {
+            entries = await strapi.entityService.findMany(collectionName, {
+                sort: { createdAt: 'DESC' },
+                populate: populateAttrib['populate'],
+            });
+        }
+        if (entries)
+        {
+            for (let s=0; s<entries.length; s++)
+            {
+                const item = entries[s];
+                const indexItemId = helper.getIndexItemId({collectionName: collectionName, itemId: item.id});    
+                const collectionConfig = await configureIndexingService.getCollectionConfig({collectionName})
+                const dataToIndex = await helper.extractDataToIndex({
+                        collectionName, data: item, collectionConfig
+                });
+                await esInterface.indexDataToSpecificIndex({itemId : indexItemId, itemData: dataToIndex}, indexName);
+            }   
+        }       
+        return true;
+    },
+    async indexPendingData() {
+        const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
+        const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
+        const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
+        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
+        const helper = strapi.plugins['elasticsearch'].services.helper;
+        const recs = await scheduleIndexingService.getItemsPendingToBeIndexed();
+        const fullSiteIndexing = recs.filter(r => r.full_site_indexing === true).length > 0
+        if (fullSiteIndexing)
+        {
+            await this.rebuildIndex();
+            for (let r=0; r< recs.length; r++)
+                await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);
+        }
+        else
+        {
+            try
+            {
+                for (let r=0; r< recs.length; r++)
+                {
+                    const col = recs[r].collection_name;
+                    if (configureIndexingService.isCollectionConfiguredToBeIndexed(col))
+                    {
+                        //Indexing the individual item
+                        if (recs[r].item_id)
+                        {
+                            if (recs[r].indexing_type !== 'remove-from-index')
+                            {
+                                const populateAttrib = helper.getPopulateAttribute({collectionName: col});     
+                                const item = await strapi.entityService.findOne(col, recs[r].item_id, {
+                                    populate: populateAttrib['populate']
+                                });
+                                const indexItemId = helper.getIndexItemId({collectionName: col, itemId: item.id});    
+                                const collectionConfig = await configureIndexingService.getCollectionConfig({collectionName: col})
+                                const dataToIndex = await helper.extractDataToIndex({
+                                        collectionName: col, data: item, collectionConfig
+                                });
+                                await esInterface.indexData({itemId : indexItemId, itemData: dataToIndex});
+                                await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);    
+                            }
+                            else
+                            {
+                                const indexItemId = helper.getIndexItemId({collectionName: col, itemId: recs[r].item_id})
+                                await esInterface.removeItemFromIndex({itemId : indexItemId});
+                                await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);    
+                            }
+                        }
+                        else //index the entire collection
+                        {
+                            //PENDING : Index an entire collection
+                            await this.indexCollection(col);
+                            await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);                      
+                        }
+                    }
+                    else
+                        await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);
+                }
+                await logIndexingService.recordIndexingPass('Indexing of ' + String(recs.length) + ' records complete.');
+            }
+            catch(err)
+            {
+                await logIndexingService.recordIndexingFail('Indexing of records failed - ' + ' ' + String(err));
+                console.log(err);
+                return false;
+            }
+        }
+        return true;
+    },
+});

package/server/services/schedule-indexing.js

@@ -0,0 +1,65 @@
+
+
+module.exports = ({ strapi }) => ({
+    async addFullSiteIndexingTask () {
+        const data = await strapi.entityService.create('plugin::elasticsearch.task', {
+            data : {
+                collection_name: '',
+                indexing_status: 'to-be-done',
+                full_site_indexing: true,
+                indexing_type: "add-to-index" 
+            }
+        });
+        return data;
+    },
+    async addCollectionToIndex({collectionUid}) {
+        const data = await strapi.entityService.create('plugin::elasticsearch.task', {
+            data : {
+                collection_name: collectionUid,
+                indexing_status: 'to-be-done',
+                full_site_indexing: false,
+                indexing_type: "add-to-index" 
+            }
+        });
+        return data;
+    },
+    async addItemToIndex({collectionUid, recordId}) {
+        const data = await strapi.entityService.create('plugin::elasticsearch.task', {
+            data : {
+                item_id: recordId, 
+                collection_name: collectionUid,
+                indexing_status: 'to-be-done',
+                full_site_indexing: false,
+                indexing_type: "add-to-index" 
+            }
+        });
+        return data;
+    },
+    async removeItemFromIndex({collectionUid, recordId}) {
+        const data = await strapi.entityService.create('plugin::elasticsearch.task', {
+            data : {
+                item_id: recordId, 
+                collection_name: collectionUid,
+                indexing_status: 'to-be-done',
+                full_site_indexing: false,
+                indexing_type: "remove-from-index"
+            }
+        });
+    },
+    async getItemsPendingToBeIndexed(){
+        const entries = await strapi.entityService.findMany('plugin::elasticsearch.task', {
+            filters: { indexing_status : 'to-be-done'},
+         });
+         return entries;     
+    },
+    async markIndexingTaskComplete (recId) {
+        const entries = await strapi.entityService.update('plugin::elasticsearch.task', recId, {
+            data : {
+                'indexing_status' : 'done'
+            }
+        }); 
+    }
+    
+    
+
+});

package/server/services/transform-content.js

@@ -0,0 +1,22 @@
+const { markdownToTxt } = require('markdown-to-txt');
+
+function transformMarkdownToText(md) {
+    let text = md;
+    try {
+        text = markdownToTxt(md);
+    }
+    catch(err) {
+        console.error('strapi-plugin-elasticsearch : Error while transforming markdown to text.');
+        console.error(err);
+    }
+    return text;
+}
+
+module.exports = {
+    transform({content, from}) {
+        if (from === 'markdown')
+            return transformMarkdownToText(content);
+        else
+            return from;
+    },
+};

package/strapi-admin.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('./admin/src').default;

package/strapi-server.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('./server');