@vegan-friendly/strapi-plugin-elasticsearch 0.2.8 → 0.3.0

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-    "version": "0.2.8",
+    "version": "0.3.0",
     "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
     "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
     "strapi": {
@@ -54,7 +54,7 @@
         }
     ],
     "engines": {
-        "node": ">=16.0.0 <=20.x.x",
+        "node": ">=16.0.0",
         "npm": ">=6.0.0"
     },
     "license": "MIT",

package/dist/server/content-types/index.d.ts

@@ -34,6 +34,10 @@ declare const _default: {
                     required: boolean;
                     default: string;
                 };
+                error_message: {
+                    type: string;
+                    required: boolean;
+                };
                 full_site_indexing: {
                     type: string;
                 };

package/dist/server/content-types/tasks.d.ts

@@ -32,6 +32,10 @@ declare const _default: {
             required: boolean;
             default: string;
         };
+        error_message: {
+            type: string;
+            required: boolean;
+        };
         full_site_indexing: {
             type: string;
         };

package/dist/server/content-types/tasks.js

@@ -14,7 +14,7 @@ exports.default = {
     },
     pluginOptions: {
         'content-manager': {
-            visible: false,
+            visible: true,
         },
         'content-type-builder': {
             visible: false,
@@ -23,17 +23,21 @@ exports.default = {
     attributes: {
         collection_name: {
             type: 'string',
-            required: true,
+            required: false,
         },
         item_id: {
             type: 'integer',
         },
         indexing_status: {
             type: 'enumeration',
-            enum: ['to-be-done', 'done'],
+            enum: ['to-be-done', 'in-progress', 'done', 'failed'],
             required: true,
             default: 'to-be-done',
         },
+        error_message: {
+            type: 'string',
+            required: false,
+        },
         full_site_indexing: {
             type: 'boolean',
         },

package/dist/server/index.d.ts

@@ -142,7 +142,9 @@ declare const _default: {
                 recordId: any;
             }): Promise<void>;
             getItemsPendingToBeIndexed(): Promise<any>;
-            markIndexingTaskComplete(recId: any): Promise<void>;
+            markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
+            markIndexingTaskInProgress(recId: any): Promise<void>;
+            getFullIndexingInProgress(): Promise<any>;
         };
         esInterface: ({ strapi }: {
             strapi: any;
@@ -150,8 +152,8 @@ declare const _default: {
         indexer: ({ strapi }: {
             strapi: any;
         }) => {
-            rebuildIndex(): Promise<boolean>;
-            indexCollection(collectionName: any, indexName?: string | null): Promise<boolean>;
+            rebuildIndex(task?: any): Promise<boolean>;
+            indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
             indexPendingData(): Promise<boolean>;
         };
         logIndexing: ({ strapi }: {
@@ -213,6 +215,10 @@ declare const _default: {
                         required: boolean;
                         default: string;
                     };
+                    error_message: {
+                        type: string;
+                        required: boolean;
+                    };
                     full_site_indexing: {
                         type: string;
                     };

package/dist/server/services/index.d.ts

@@ -38,7 +38,9 @@ declare const _default: {
             recordId: any;
         }): Promise<void>;
         getItemsPendingToBeIndexed(): Promise<any>;
-        markIndexingTaskComplete(recId: any): Promise<void>;
+        markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
+        markIndexingTaskInProgress(recId: any): Promise<void>;
+        getFullIndexingInProgress(): Promise<any>;
     };
     esInterface: ({ strapi }: {
         strapi: any;
@@ -46,8 +48,8 @@ declare const _default: {
     indexer: ({ strapi }: {
         strapi: any;
     }) => {
-        rebuildIndex(): Promise<boolean>;
-        indexCollection(collectionName: any, indexName?: string | null): Promise<boolean>;
+        rebuildIndex(task?: any): Promise<boolean>;
+        indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
         indexPendingData(): Promise<boolean>;
     };
     logIndexing: ({ strapi }: {

package/dist/server/services/perform-indexing.d.ts

@@ -1,8 +1,8 @@
 declare const _default: ({ strapi }: {
     strapi: any;
 }) => {
-    rebuildIndex(): Promise<boolean>;
-    indexCollection(collectionName: any, indexName?: string | null): Promise<boolean>;
+    rebuildIndex(task?: any): Promise<boolean>;
+    indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
     indexPendingData(): Promise<boolean>;
 };
 export default _default;

package/dist/server/services/perform-indexing.js

@@ -1,32 +1,54 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = ({ strapi }) => ({
-    async rebuildIndex() {
+    async rebuildIndex(task = null) {
         const helper = strapi.plugins['elasticsearch'].services.helper;
         const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
         const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
         const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
         const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
         const virtualCollectionsIndexer = strapi.plugins['elasticsearch'].services['virtualCollectionsIndexer'];
+        const virtualCollectionsRegistry = strapi.plugins['elasticsearch'].services['virtualCollectionsRegistry'];
+        let taskError = null;
         try {
             console.log('strapi-plugin-elasticsearch : Request to rebuild the index received.');
+            const fullIndexingInProgress = await scheduleIndexingService.getFullIndexingInProgress();
+            if (fullIndexingInProgress.length > 0) {
+                const msg = `Indexing is already in progress - see tasks ${fullIndexingInProgress.map((t) => t.id)}. This request is ignored and marked as failed.`;
+                console.log('strapi-plugin-elasticsearch : ' + msg);
+                await logIndexingService.recordIndexingFail(msg);
+                return false;
+            }
+            const cols = await configureIndexingService.getCollectionsConfiguredForIndexing();
+            const needsNewIndex = cols.length > 0 || virtualCollectionsRegistry.getAll().some((vc) => vc.indexAlias == null);
             const oldIndexName = await helper.getCurrentIndexName();
             console.log('strapi-plugin-elasticsearch : Recording the previous index name : ', oldIndexName);
             //Step 1 : Create a new index
-            const newIndexName = await helper.getIncrementedIndexName();
-            await esInterface.createIndex(newIndexName);
-            console.log('strapi-plugin-elasticsearch : Created new index with name : ', newIndexName);
+            let newIndexName;
+            if (needsNewIndex) {
+                newIndexName = await helper.getIncrementedIndexName();
+                await esInterface.createIndex(newIndexName);
+                console.log('strapi-plugin-elasticsearch : Created new index with name : ', newIndexName);
+            }
+            else {
+                newIndexName = oldIndexName;
+                console.log('strapi-plugin-elasticsearch : No need to create new index, as there are no collections to re-index, and no virtual-collections that use the default index. sticking to current index:', newIndexName);
+            }
             //Step 2 : Index all the stuff on this new index
             console.log('strapi-plugin-elasticsearch : Starting to index all data into the new index.');
-            const item = await scheduleIndexingService.addFullSiteIndexingTask();
-            if (item.id) {
-                const cols = await configureIndexingService.getCollectionsConfiguredForIndexing();
-                for (let r = 0; r < cols.length; r++)
-                    await this.indexCollection(cols[r], newIndexName);
+            if (task == null) {
+                task = await scheduleIndexingService.addFullSiteIndexingTask();
+            }
+            if (task?.id) {
+                await scheduleIndexingService.markIndexingTaskInProgress(task.id);
+                let entitiesIndexed = 0;
+                for (let r = 0; r < cols.length; r++) {
+                    entitiesIndexed += await this.indexCollection(cols[r], newIndexName);
+                }
                 // Indexing the virtual collections
-                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections.');
-                const totalIndexed = await virtualCollectionsIndexer.reindexAll(newIndexName);
-                await scheduleIndexingService.markIndexingTaskComplete(item.id);
+                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections. task id : ', task.id);
+                const virtualEntriesIndexed = await virtualCollectionsIndexer.reindexAll(newIndexName);
+                await scheduleIndexingService.markIndexingTaskComplete(task.id);
                 console.log('strapi-plugin-elasticsearch : Indexing of data into the new index complete.');
                 //Step 4 : Move the alias to this new index
                 await esInterface.attachAliasToIndex(newIndexName);
@@ -34,7 +56,7 @@ exports.default = ({ strapi }) => ({
                 console.log('strapi-plugin-elasticsearch : Deleting the previous indices');
                 //Step 5 : Delete the previous index
                 await helper.deleteOldIndices();
-                await logIndexingService.recordIndexingPass('Request to immediately re-index site-wide content completed successfully.');
+                await logIndexingService.recordIndexingPass(`Re-index site-wide content completed successfully. ${entitiesIndexed} entries indexed. ${virtualEntriesIndexed} virtual entries indexed.`);
                 return true;
             }
             else {
@@ -43,11 +65,17 @@ exports.default = ({ strapi }) => ({
             }
         }
         catch (err) {
+            taskError = err.message || String(err);
             console.log('strapi-plugin-elasticsearch : searchController : An error was encountered while re-indexing.');
             console.log(err);
             await logIndexingService.recordIndexingFail(err);
             throw err;
         }
+        finally {
+            if (task?.id) {
+                await scheduleIndexingService.markIndexingTaskComplete(task.id, taskError);
+            }
+        }
     },
     async indexCollection(collectionName, indexName = null) {
         const helper = strapi.plugins['elasticsearch'].services.helper;
@@ -93,7 +121,7 @@ exports.default = ({ strapi }) => ({
                 await esInterface.indexDataToSpecificIndex({ itemId: indexItemId, itemData: dataToIndex }, indexName);
             }
         }
-        return true;
+        return entries.length ?? 0;
     },
     async indexPendingData() {
         const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
@@ -103,17 +131,22 @@ exports.default = ({ strapi }) => ({
         const helper = strapi.plugins['elasticsearch'].services.helper;
         const indexAlias = await strapi.config.get('plugin.elasticsearch').indexAliasName;
         const recs = await scheduleIndexingService.getItemsPendingToBeIndexed();
-        const fullSiteIndexing = recs.filter((r) => r.full_site_indexing === true).length > 0;
+        const fullSiteIndexTasks = recs.filter((r) => r.full_site_indexing === true);
+        const fullSiteIndexing = fullSiteIndexTasks.length > 0;
         if (fullSiteIndexing) {
-            await this.rebuildIndex();
-            for (let r = 0; r < recs.length; r++)
-                await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);
+            const success = await this.rebuildIndex(fullSiteIndexTasks[0]);
+            if (success) {
+                // Mark all pending tasks as complete, as they are implicitly covered by the full-site indexing.
+                for (let r = 0; r < recs.length; r++)
+                    await scheduleIndexingService.markIndexingTaskComplete(recs[r].id);
+            }
         }
         else {
             try {
                 for (let r = 0; r < recs.length; r++) {
                     const col = recs[r].collection_name;
                     if (configureIndexingService.isCollectionConfiguredToBeIndexed(col)) {
+                        await scheduleIndexingService.markIndexingTaskInProgress(recs[r].id);
                         //Indexing the individual item
                         if (recs[r].item_id) {
                             if (recs[r].indexing_type !== 'remove-from-index') {

package/dist/server/services/schedule-indexing.d.ts

@@ -14,6 +14,8 @@ declare const _default: ({ strapi }: {
         recordId: any;
     }): Promise<void>;
     getItemsPendingToBeIndexed(): Promise<any>;
-    markIndexingTaskComplete(recId: any): Promise<void>;
+    markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
+    markIndexingTaskInProgress(recId: any): Promise<void>;
+    getFullIndexingInProgress(): Promise<any>;
 };
 export default _default;

package/dist/server/services/schedule-indexing.js

@@ -52,11 +52,29 @@ exports.default = ({ strapi }) => ({
         });
         return entries;
     },
-    async markIndexingTaskComplete(recId) {
+    async markIndexingTaskComplete(recId, error = null) {
+        const status = error ? 'failed' : 'done';
         const entries = await strapi.entityService.update('plugin::elasticsearch.task', recId, {
             data: {
-                indexing_status: 'done',
+                indexing_status: status,
+                error_message: error,
             },
         });
     },
+    async markIndexingTaskInProgress(recId) {
+        await strapi.entityService.update('plugin::elasticsearch.task', recId, {
+            data: {
+                indexing_status: 'in-progress',
+            },
+        });
+    },
+    async getFullIndexingInProgress() {
+        const entries = await strapi.entityService.findMany('plugin::elasticsearch.task', {
+            filters: {
+                indexing_status: 'in-progress',
+                full_site_indexing: true,
+            },
+        });
+        return entries;
+    },
 });

package/dist/server/services/virtual-collections-indexer.js

@@ -24,7 +24,9 @@ exports.default = ({ strapi }) => {
             try {
                 const results = await collection.extractByIds([itemId]);
                 if (!results || !Array.isArray(results) || results.length === 0) {
-                    strapi.log.warn(`No data extracted for ${collectionName} with ID ${itemId}`);
+                    // item does not exit - delete it from index
+                    await this.deleteItem(collectionName, itemId);
+                    strapi.log.debug(`Deleted virtual item: ${collectionName}:${itemId}`);
                     return null;
                 }
                 const itemData = results[0];
@@ -58,11 +60,13 @@ exports.default = ({ strapi }) => {
         async reindex(collection) {
             const collectionName = collection.collectionName;
             const privateIndexAlias = collection.indexAlias;
+            const pageSize = 100;
             const helper = getHelperService();
             let timestamp = Date.now();
+            let indexName = '';
+            let errors = 0;
             try {
                 const esInterface = getElasticsearchService();
-                let indexName;
                 if (privateIndexAlias) {
                     indexName = await helper.getIncrementedIndexName(privateIndexAlias);
                     await esInterface.createIndex(indexName, collection.mappings);
@@ -71,25 +75,53 @@ exports.default = ({ strapi }) => {
                     indexName = await helper.getCurrentIndexName();
                 }
                 let page = 0;
-                let hasMoreData = true;
+                let prevPageData = [];
                 let totalIndexed = 0;
-                while (hasMoreData) {
-                    const pageData = await collection.extractData(page);
+                const pageLimit = 10000;
+                while (page <= pageLimit) {
+                    let pageData;
+                    try {
+                        pageData = await collection.extractData(page, pageSize);
+                    }
+                    catch (error) {
+                        strapi.log.error(`Error extracting data for page ${page} of ${collectionName}: ${error.message}`);
+                        errors += pageSize;
+                        page++;
+                        continue;
+                    }
+                    strapi.log.debug(`Extracted ${pageData.length} items from ${collectionName} for page ${page}`);
                     if (!Array.isArray(pageData) || pageData.length === 0) {
-                        hasMoreData = false;
                         break;
                     }
+                    if (JSON.stringify(prevPageData) == JSON.stringify(pageData)) {
+                        throw new Error(`Infinite loop detected at page ${page} while reindexing ${collectionName}. Stopping reindexing. Check this virtual-collection's extractData().
+  current page 1st item (id ${pageData[0]?.id}):
+  ${JSON.stringify(pageData[0])}
+  prev page 1st item (id ${prevPageData[0]?.id}):
+  ${JSON.stringify(prevPageData[0])}`);
+                    }
+                    if (page >= pageLimit) {
+                        strapi.log.warn(`Page ${page} of ${collectionName} is greater than page-limit (${pageLimit}). stopping indexing this virtual-collection.`);
+                        pageData.length = pageLimit;
+                    }
                     const operations = [];
                     for (const itemData of pageData) {
                         const itemId = collection.getIndexItemId(itemData.id, collectionName);
                         operations.push({ itemId, itemData });
                     }
                     if (operations.length > 0) {
-                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName)));
+                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName).catch((err) => {
+                            strapi.log.error(`Failed to index item ${op.itemId} in ${collectionName}: ${err}`);
+                            errors++;
+                        })));
                     }
                     totalIndexed += pageData.length;
+                    prevPageData = pageData;
                     page++;
                 }
+                if (errors > 0) {
+                    throw new Error(`Failed to index ${errors} of ${totalIndexed} items for virtual collection ${collectionName}. Errors were logged. Alias was not updated. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                }
                 strapi.log.info(`Reindexed ${totalIndexed} items for virtual collection: ${collectionName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}. now updating alias.`);
                 if (privateIndexAlias) {
                     timestamp = Date.now();
@@ -102,7 +134,7 @@ exports.default = ({ strapi }) => {
                 return totalIndexed;
             }
             catch (error) {
-                strapi.log.error(`Error reindexing ${collectionName}: ${error?.message} after ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                strapi.log.error(`Error reindexing ${collectionName} to index ${indexName}: ${error?.message} after ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
                 throw error;
             }
         },
@@ -125,14 +157,7 @@ exports.default = ({ strapi }) => {
                 const idsToReindex = await trigger.getIdsToReindex(result);
                 // Reindex each item
                 for (const id of idsToReindex) {
-                    const isDelete = event.action?.toLowerCase()?.includes('delete') && trigger.alsoTriggerDelete && id === result.id;
-                    if (isDelete) {
-                        //delete the item from the index, if the item being delete is the one being reindexed
-                        await this.deleteItem(collection.collectionName, id);
-                    }
-                    else {
-                        await this.indexItem(collection.collectionName, id);
-                    }
+                    await this.indexItem(collection.collectionName, id);
                 }
             }
         },

package/dist/server/services/virtual-collections-registry.js

@@ -47,10 +47,9 @@ const configSchema = yup.object({
         .of(yup.object({
         collection: yup.string().required(),
         getIdsToReindex: isFunction().required(),
-        alsoTriggerDelete: yup.boolean().default(false),
     }))
         .default([]),
-    mappings: yup.object().default({}),
+    mappings: yup.object().optional(),
 });
 /**
  * Service to handle indexing of virtual collections
@@ -73,7 +72,7 @@ exports.default = ({ strapi }) => {
                 const virtualCollectionsFactories = strapi.plugin('elasticsearch').config('virtualCollections') || [];
                 config = virtualCollectionsFactories.map((factory) => {
                     let collectionConfig = factory(strapi);
-                    collectionConfig = configSchema.validateSync(collectionConfig, { stripUnknown: true });
+                    collectionConfig = configSchema.validateSync(collectionConfig);
                     collectionConfig.getIndexItemId = collectionConfig.getIndexItemId || ((id) => helper.getIndexItemId({ collectionName: collectionConfig.collectionName, itemId: id }));
                     return { ...defaultConf, ...collectionConfig };
                 });

package/dist/server/types/virtual-collections.type.d.ts

@@ -45,11 +45,6 @@ export type VirtualCollectionConfig = {
          * @returns ids of the items to be reindexed.
          */
         getIdsToReindex: GetIdsToIndexFunction;
-        /**
-         * if true, and the trigger is a delete event, the item of the virtual collection will be deleted as well if the id returned from getIdsToReindex match.
-         * defaults to false.
-         */
-        alsoTriggerDelete?: boolean;
     }>;
     /**
      * Optional schema to be sent to Elasticsearch when creating the index.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.2.8",
+  "version": "0.3.0",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {
@@ -54,7 +54,7 @@
     }
   ],
   "engines": {
-    "node": ">=16.0.0 <=20.x.x",
+    "node": ">=16.0.0",
     "npm": ">=6.0.0"
   },
   "license": "MIT",
@@ -64,4 +64,4 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.8.3"
   }
-}
+}