@vegan-friendly/strapi-plugin-elasticsearch 0.2.7 → 0.2.9

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-    "version": "0.2.7",
+    "version": "0.2.9",
     "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
     "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
     "strapi": {

package/dist/server/bootstrap.js

@@ -25,7 +25,12 @@ exports.default = async ({ strapi }) => {
             strapi.cron.add({
                 elasticsearchIndexing: {
                     task: async ({ strapi }) => {
-                        await indexer.indexPendingData();
+                        try {
+                            await indexer.indexPendingData();
+                        }
+                        catch (err) {
+                            strapi.log.error('Error while indexing data: ', err);
+                        }
                     },
                     options: {
                         rule: pluginConfig['indexingCronSchedule'],
@@ -119,6 +124,15 @@ exports.default = async ({ strapi }) => {
         const registry = strapi.service('plugin::elasticsearch.virtualCollectionsRegistry');
         // Setup lifecycle hooks
         const virtualCollections = registry.getAll();
+        // Check if indices exists, if not create them
+        virtualCollections.forEach(async (collection) => {
+            const indexName = await helper.getCurrentIndexName(collection.indexAlias);
+            const indexExists = await esInterface.listIndicesByPattern(indexName);
+            if (!indexExists.includes(indexName)) {
+                await esInterface.createIndex(indexName, collection.mappings);
+                strapi.log.info(`Created Elasticsearch index: ${indexName}`);
+            }
+        });
         // Create a set of all collections that need hooks
         const collectionsToHook = new Set();
         virtualCollections.forEach((collection) => {

package/dist/server/content-types/tasks.js

@@ -14,7 +14,7 @@ exports.default = {
     },
     pluginOptions: {
         'content-manager': {
-            visible: false,
+            visible: true,
         },
         'content-type-builder': {
             visible: false,
@@ -23,14 +23,14 @@ exports.default = {
     attributes: {
         collection_name: {
             type: 'string',
-            required: true,
+            required: false,
         },
         item_id: {
             type: 'integer',
         },
         indexing_status: {
             type: 'enumeration',
-            enum: ['to-be-done', 'done'],
+            enum: ['to-be-done', 'in-progress', 'done'],
             required: true,
             default: 'to-be-done',
         },

package/dist/server/index.d.ts

@@ -143,6 +143,8 @@ declare const _default: {
             }): Promise<void>;
             getItemsPendingToBeIndexed(): Promise<any>;
             markIndexingTaskComplete(recId: any): Promise<void>;
+            markIndexingTaskInProgress(recId: any): Promise<void>;
+            getFullIndexingInProgress(): Promise<any>;
         };
         esInterface: ({ strapi }: {
             strapi: any;
@@ -150,8 +152,8 @@ declare const _default: {
         indexer: ({ strapi }: {
             strapi: any;
         }) => {
-            rebuildIndex(): Promise<boolean>;
-            indexCollection(collectionName: any, indexName?: string | null): Promise<boolean>;
+            rebuildIndex(item?: any): Promise<boolean>;
+            indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
             indexPendingData(): Promise<boolean>;
         };
         logIndexing: ({ strapi }: {

package/dist/server/services/es-interface.js

@@ -156,10 +156,18 @@ exports.default = ({ strapi }) => ({
         }
     },
     async listIndicesByPattern(pattern = 'restaurants*') {
-        const results = await client.cat.indices({
-            index: pattern,
-            format: 'json',
-        });
-        return results.map((index) => index.index).filter((index) => index != null);
+        try {
+            const results = await client.cat.indices({
+                index: pattern,
+                format: 'json',
+            });
+            return results.map((index) => index.index).filter((index) => index != null);
+        }
+        catch (err) {
+            if (err?.message?.includes('index_not_found_exception')) {
+                return [];
+            }
+            throw err;
+        }
     },
 });

package/dist/server/services/index.d.ts

@@ -39,6 +39,8 @@ declare const _default: {
         }): Promise<void>;
         getItemsPendingToBeIndexed(): Promise<any>;
         markIndexingTaskComplete(recId: any): Promise<void>;
+        markIndexingTaskInProgress(recId: any): Promise<void>;
+        getFullIndexingInProgress(): Promise<any>;
     };
     esInterface: ({ strapi }: {
         strapi: any;
@@ -46,8 +48,8 @@ declare const _default: {
     indexer: ({ strapi }: {
         strapi: any;
     }) => {
-        rebuildIndex(): Promise<boolean>;
-        indexCollection(collectionName: any, indexName?: string | null): Promise<boolean>;
+        rebuildIndex(item?: any): Promise<boolean>;
+        indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
         indexPendingData(): Promise<boolean>;
     };
     logIndexing: ({ strapi }: {

package/dist/server/services/perform-indexing.d.ts

@@ -1,8 +1,8 @@
 declare const _default: ({ strapi }: {
     strapi: any;
 }) => {
-    rebuildIndex(): Promise<boolean>;
-    indexCollection(collectionName: any, indexName?: string | null): Promise<boolean>;
+    rebuildIndex(item?: any): Promise<boolean>;
+    indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
     indexPendingData(): Promise<boolean>;
 };
 export default _default;

package/dist/server/services/perform-indexing.js

@@ -1,31 +1,52 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = ({ strapi }) => ({
-    async rebuildIndex() {
+    async rebuildIndex(item = null) {
         const helper = strapi.plugins['elasticsearch'].services.helper;
         const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
         const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
         const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
         const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
         const virtualCollectionsIndexer = strapi.plugins['elasticsearch'].services['virtualCollectionsIndexer'];
+        const virtualCollectionsRegistry = strapi.plugins['elasticsearch'].services['virtualCollectionsRegistry'];
         try {
             console.log('strapi-plugin-elasticsearch : Request to rebuild the index received.');
+            const fullIndexingInProgress = await scheduleIndexingService.getFullIndexingInProgress();
+            if (fullIndexingInProgress.length > 0) {
+                const msg = `Indexing is already in progress - see tasks ${fullIndexingInProgress.map((t) => t.id)}. This request is ignored and marked as failed.`;
+                console.log('strapi-plugin-elasticsearch : ' + msg);
+                await logIndexingService.recordIndexingFail(msg);
+                return false;
+            }
+            const cols = await configureIndexingService.getCollectionsConfiguredForIndexing();
+            const needsNewIndex = cols.length > 0 || virtualCollectionsRegistry.getAll().some((vc) => vc.indexAlias == null);
             const oldIndexName = await helper.getCurrentIndexName();
             console.log('strapi-plugin-elasticsearch : Recording the previous index name : ', oldIndexName);
             //Step 1 : Create a new index
-            const newIndexName = await helper.getIncrementedIndexName();
-            await esInterface.createIndex(newIndexName);
-            console.log('strapi-plugin-elasticsearch : Created new index with name : ', newIndexName);
+            let newIndexName;
+            if (needsNewIndex) {
+                newIndexName = await helper.getIncrementedIndexName();
+                await esInterface.createIndex(newIndexName);
+                console.log('strapi-plugin-elasticsearch : Created new index with name : ', newIndexName);
+            }
+            else {
+                newIndexName = oldIndexName;
+                console.log('strapi-plugin-elasticsearch : No need to create new index, as there are no collections to re-index, and no virtual-collections that use the default index. sticking to current index:', newIndexName);
+            }
             //Step 2 : Index all the stuff on this new index
             console.log('strapi-plugin-elasticsearch : Starting to index all data into the new index.');
-            const item = await scheduleIndexingService.addFullSiteIndexingTask();
-            if (item.id) {
-                const cols = await configureIndexingService.getCollectionsConfiguredForIndexing();
-                for (let r = 0; r < cols.length; r++)
-                    await this.indexCollection(cols[r], newIndexName);
+            if (item == null) {
+                item = await scheduleIndexingService.addFullSiteIndexingTask();
+            }
+            if (item?.id) {
+                await scheduleIndexingService.markIndexingTaskInProgress(item.id);
+                let entitiesIndexed = 0;
+                for (let r = 0; r < cols.length; r++) {
+                    entitiesIndexed += await this.indexCollection(cols[r], newIndexName);
+                }
                 // Indexing the virtual collections
-                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections.');
-                const totalIndexed = await virtualCollectionsIndexer.reindexAll(newIndexName);
+                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections. task id : ', item.id);
+                const virtualEntriesIndexed = await virtualCollectionsIndexer.reindexAll(newIndexName);
                 await scheduleIndexingService.markIndexingTaskComplete(item.id);
                 console.log('strapi-plugin-elasticsearch : Indexing of data into the new index complete.');
                 //Step 4 : Move the alias to this new index
@@ -34,7 +55,7 @@ exports.default = ({ strapi }) => ({
                 console.log('strapi-plugin-elasticsearch : Deleting the previous indices');
                 //Step 5 : Delete the previous index
                 await helper.deleteOldIndices();
-                await logIndexingService.recordIndexingPass('Request to immediately re-index site-wide content completed successfully.');
+                await logIndexingService.recordIndexingPass(`Re-index site-wide content completed successfully. ${entitiesIndexed} entries indexed. ${virtualEntriesIndexed} virtual entries indexed.`);
                 return true;
             }
             else {
@@ -48,6 +69,11 @@ exports.default = ({ strapi }) => ({
             await logIndexingService.recordIndexingFail(err);
             throw err;
         }
+        finally {
+            if (item?.id) {
+                await scheduleIndexingService.markIndexingTaskComplete(item.id);
+            }
+        }
     },
     async indexCollection(collectionName, indexName = null) {
         const helper = strapi.plugins['elasticsearch'].services.helper;
@@ -93,7 +119,7 @@ exports.default = ({ strapi }) => ({
                 await esInterface.indexDataToSpecificIndex({ itemId: indexItemId, itemData: dataToIndex }, indexName);
             }
         }
-        return true;
+        return entries.length ?? 0;
     },
     async indexPendingData() {
         const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
@@ -103,17 +129,22 @@ exports.default = ({ strapi }) => ({
         const helper = strapi.plugins['elasticsearch'].services.helper;
         const indexAlias = await strapi.config.get('plugin.elasticsearch').indexAliasName;
         const recs = await scheduleIndexingService.getItemsPendingToBeIndexed();
-        const fullSiteIndexing = recs.filter((r) => r.full_site_indexing === true).length > 0;
+        const fullSiteIndexTasks = recs.filter((r) => r.full_site_indexing === true);
+        const fullSiteIndexing = fullSiteIndexTasks.length > 0;
         if (fullSiteIndexing) {
-            await this.rebuildIndex();
-            for (let r = 0; r < recs.length; r++)
-                await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);
+            const success = await this.rebuildIndex(fullSiteIndexTasks[0]);
+            if (success) {
+                // Mark all pending tasks as complete, as they are implicitly covered by the full-site indexing.
+                for (let r = 0; r < recs.length; r++)
+                    await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);
+            }
         }
         else {
             try {
                 for (let r = 0; r < recs.length; r++) {
                     const col = recs[r].collection_name;
                     if (configureIndexingService.isCollectionConfiguredToBeIndexed(col)) {
+                        await scheduleIndexingService.markIndexingTaskInProgress(recs[r].id);
                         //Indexing the individual item
                         if (recs[r].item_id) {
                             if (recs[r].indexing_type !== 'remove-from-index') {

package/dist/server/services/schedule-indexing.d.ts

@@ -15,5 +15,7 @@ declare const _default: ({ strapi }: {
     }): Promise<void>;
     getItemsPendingToBeIndexed(): Promise<any>;
     markIndexingTaskComplete(recId: any): Promise<void>;
+    markIndexingTaskInProgress(recId: any): Promise<void>;
+    getFullIndexingInProgress(): Promise<any>;
 };
 export default _default;

package/dist/server/services/schedule-indexing.js

@@ -59,4 +59,20 @@ exports.default = ({ strapi }) => ({
             },
         });
     },
+    async markIndexingTaskInProgress(recId) {
+        await strapi.entityService.update('plugin::elasticsearch.task', recId, {
+            data: {
+                indexing_status: 'in-progress',
+            },
+        });
+    },
+    async getFullIndexingInProgress() {
+        const entries = await strapi.entityService.findMany('plugin::elasticsearch.task', {
+            filters: {
+                indexing_status: 'in-progress',
+                full_site_indexing: true,
+            },
+        });
+        return entries;
+    },
 });

package/dist/server/services/virtual-collections-indexer.js

@@ -60,9 +60,9 @@ exports.default = ({ strapi }) => {
             const privateIndexAlias = collection.indexAlias;
             const helper = getHelperService();
             let timestamp = Date.now();
+            let indexName = '';
             try {
                 const esInterface = getElasticsearchService();
-                let indexName;
                 if (privateIndexAlias) {
                     indexName = await helper.getIncrementedIndexName(privateIndexAlias);
                     await esInterface.createIndex(indexName, collection.mappings);
@@ -71,14 +71,26 @@ exports.default = ({ strapi }) => {
                     indexName = await helper.getCurrentIndexName();
                 }
                 let page = 0;
-                let hasMoreData = true;
+                let prevPageData = [];
                 let totalIndexed = 0;
-                while (hasMoreData) {
+                const pageLimit = 10000;
+                while (page <= pageLimit) {
                     const pageData = await collection.extractData(page);
+                    strapi.log.debug(`Extracted ${pageData.length} items from ${collectionName} for page ${page}`);
                     if (!Array.isArray(pageData) || pageData.length === 0) {
-                        hasMoreData = false;
                         break;
                     }
+                    if (JSON.stringify(prevPageData) == JSON.stringify(pageData)) {
+                        throw new Error(`Infinite loop detected at page ${page} while reindexing ${collectionName}. Stopping reindexing. Check this virtual-collection's extractData().
+  current page 1st item (id ${pageData[0]?.id}):
+  ${JSON.stringify(pageData[0])}
+  prev page 1st item (id ${prevPageData[0]?.id}):
+  ${JSON.stringify(prevPageData[0])}`);
+                    }
+                    if (page >= pageLimit) {
+                        strapi.log.warn(`Page ${page} of ${collectionName} is greater than page-limit (${pageLimit}). stopping indexing this virtual-collection.`);
+                        pageData.length = pageLimit;
+                    }
                     const operations = [];
                     for (const itemData of pageData) {
                         const itemId = collection.getIndexItemId(itemData.id, collectionName);
@@ -88,6 +100,7 @@ exports.default = ({ strapi }) => {
                         await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName)));
                     }
                     totalIndexed += pageData.length;
+                    prevPageData = pageData;
                     page++;
                 }
                 strapi.log.info(`Reindexed ${totalIndexed} items for virtual collection: ${collectionName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}. now updating alias.`);
@@ -102,7 +115,7 @@ exports.default = ({ strapi }) => {
                 return totalIndexed;
             }
             catch (error) {
-                strapi.log.error(`Error reindexing ${collectionName}: ${error?.message} after ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                strapi.log.error(`Error reindexing ${collectionName} to index ${indexName}: ${error?.message} after ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
                 throw error;
             }
         },

package/dist/server/services/virtual-collections-registry.js

@@ -36,20 +36,21 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const yup = __importStar(require("yup"));
 const isFunction = () => yup.mixed().test('is-function', `must be a function`, (value) => typeof value === 'function');
 const configSchema = yup.object({
-    indexAlias: yup.string().nullable(),
+    indexAlias: yup.string().notRequired().nonNullable(),
     collectionName: yup.string().required(),
     extractData: isFunction().required(),
     extractByIds: isFunction().required(),
     getIndexItemId: isFunction(),
     triggers: yup
         .array()
+        .optional()
         .of(yup.object({
         collection: yup.string().required(),
-        getIdsToReindex: isFunction(),
+        getIdsToReindex: isFunction().required(),
         alsoTriggerDelete: yup.boolean().default(false),
     }))
         .default([]),
-    mappings: yup.object().default({}),
+    mappings: yup.object().optional(),
 });
 /**
  * Service to handle indexing of virtual collections
@@ -69,9 +70,10 @@ exports.default = ({ strapi }) => {
             if (!config) {
                 const helper = strapi.plugin('elasticsearch').service('helper');
                 const defaultConf = configSchema.getDefault();
-                config = strapi.plugin('elasticsearch').config('virtualCollections') || [];
-                config = config.map((collection) => {
-                    const collectionConfig = configSchema.validateSync(collection, { strict: true });
+                const virtualCollectionsFactories = strapi.plugin('elasticsearch').config('virtualCollections') || [];
+                config = virtualCollectionsFactories.map((factory) => {
+                    let collectionConfig = factory(strapi);
+                    collectionConfig = configSchema.validateSync(collectionConfig);
                     collectionConfig.getIndexItemId = collectionConfig.getIndexItemId || ((id) => helper.getIndexItemId({ collectionName: collectionConfig.collectionName, itemId: id }));
                     return { ...defaultConf, ...collectionConfig };
                 });

package/dist/server/types/virtual-collections.type.d.ts

@@ -1,4 +1,9 @@
 import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+export type VirtualCollectionFactory = (strapi: any) => VirtualCollectionConfig;
+export type ExtractByIdsFunction = (ids: number[]) => Promise<StrapiEntity[]>;
+export type ExtractDataFunction = (page: number, pageSize?: number) => Promise<StrapiEntity[]>;
+export type GetIndexItemIdFunction = (itemId: number, collectionName: string) => string;
+export type GetIdsToIndexFunction = (event: any) => Promise<number[]>;
 export type VirtualCollectionConfig = {
     /**
      * Optional -
@@ -17,8 +22,8 @@ export type VirtualCollectionConfig = {
      * e.g 'api::restaurants.restaurants'.
      */
     collectionName: string;
-    extractData: (page: number, pageSize?: number) => Promise<StrapiEntity[]>;
-    extractByIds: (ids: number[]) => Promise<StrapiEntity[]>;
+    extractData: ExtractDataFunction;
+    extractByIds: ExtractByIdsFunction;
     /**
      *  Optional -
      * A function that takes an item and returns the id of the item to be used in the index.
@@ -28,7 +33,7 @@ export type VirtualCollectionConfig = {
      * @param collectionName collection name. you probably want to use this to create a unique id for the item, especially if it is saved to the default index.
      * @returns the id of the item to be used in the index, _id. must be unique accross the index.
      */
-    getIndexItemId?: (itemId: number, collectionName: string) => string;
+    getIndexItemId?: GetIndexItemIdFunction;
     triggers: Array<{
         /**
          * collection name to listen to for changes.
@@ -39,7 +44,7 @@ export type VirtualCollectionConfig = {
          * @param event - The event object containing the data to be indexed.
          * @returns ids of the items to be reindexed.
          */
-        getIdsToReindex: (event: any) => Promise<number[]>;
+        getIdsToReindex: GetIdsToIndexFunction;
         /**
          * if true, and the trigger is a delete event, the item of the virtual collection will be deleted as well if the id returned from getIdsToReindex match.
          * defaults to false.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {