@vegan-friendly/strapi-plugin-elasticsearch 0.2.9 → 0.3.1

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-    "version": "0.2.9",
+    "version": "0.3.1",
     "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
     "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
     "strapi": {
@@ -54,7 +54,7 @@
         }
     ],
     "engines": {
-        "node": ">=16.0.0 <=20.x.x",
+        "node": ">=16.0.0",
         "npm": ">=6.0.0"
     },
     "license": "MIT",

package/dist/server/bootstrap.js

@@ -156,6 +156,9 @@ exports.default = async ({ strapi }) => {
                 },
             });
         });
+        // clean up old indexing tasks, as server is booting.
+        // allow 60 seconds, in case strapi is being run in cluster mode and this is the second instance
+        await scheduleIndexingService.getActiveFullIndexingTasks(60);
         configureIndexingService.markInitialized();
     }
     catch (err) {

package/dist/server/content-types/index.d.ts

@@ -34,6 +34,10 @@ declare const _default: {
                     required: boolean;
                     default: string;
                 };
+                error_message: {
+                    type: string;
+                    required: boolean;
+                };
                 full_site_indexing: {
                     type: string;
                 };

package/dist/server/content-types/tasks.d.ts

@@ -32,6 +32,10 @@ declare const _default: {
             required: boolean;
             default: string;
         };
+        error_message: {
+            type: string;
+            required: boolean;
+        };
         full_site_indexing: {
             type: string;
         };

package/dist/server/content-types/tasks.js

@@ -30,10 +30,14 @@ exports.default = {
         },
         indexing_status: {
             type: 'enumeration',
-            enum: ['to-be-done', 'in-progress', 'done'],
+            enum: ['to-be-done', 'in-progress', 'done', 'failed'],
             required: true,
             default: 'to-be-done',
         },
+        error_message: {
+            type: 'string',
+            required: false,
+        },
         full_site_indexing: {
             type: 'boolean',
         },

package/dist/server/index.d.ts

@@ -142,9 +142,9 @@ declare const _default: {
                 recordId: any;
             }): Promise<void>;
             getItemsPendingToBeIndexed(): Promise<any>;
-            markIndexingTaskComplete(recId: any): Promise<void>;
+            markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
             markIndexingTaskInProgress(recId: any): Promise<void>;
-            getFullIndexingInProgress(): Promise<any>;
+            getActiveFullIndexingTasks(staleTaskThresholdSeconds?: number): Promise<any[]>;
         };
         esInterface: ({ strapi }: {
             strapi: any;
@@ -152,7 +152,7 @@ declare const _default: {
         indexer: ({ strapi }: {
             strapi: any;
         }) => {
-            rebuildIndex(item?: any): Promise<boolean>;
+            rebuildIndex(task?: any): Promise<boolean>;
             indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
             indexPendingData(): Promise<boolean>;
         };
@@ -215,6 +215,10 @@ declare const _default: {
                         required: boolean;
                         default: string;
                     };
+                    error_message: {
+                        type: string;
+                        required: boolean;
+                    };
                     full_site_indexing: {
                         type: string;
                     };

package/dist/server/services/index.d.ts

@@ -38,9 +38,9 @@ declare const _default: {
             recordId: any;
         }): Promise<void>;
         getItemsPendingToBeIndexed(): Promise<any>;
-        markIndexingTaskComplete(recId: any): Promise<void>;
+        markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
         markIndexingTaskInProgress(recId: any): Promise<void>;
-        getFullIndexingInProgress(): Promise<any>;
+        getActiveFullIndexingTasks(staleTaskThresholdSeconds?: number): Promise<any[]>;
     };
     esInterface: ({ strapi }: {
         strapi: any;
@@ -48,7 +48,7 @@ declare const _default: {
     indexer: ({ strapi }: {
         strapi: any;
     }) => {
-        rebuildIndex(item?: any): Promise<boolean>;
+        rebuildIndex(task?: any): Promise<boolean>;
         indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
         indexPendingData(): Promise<boolean>;
     };

package/dist/server/services/perform-indexing.d.ts

@@ -1,7 +1,7 @@
 declare const _default: ({ strapi }: {
     strapi: any;
 }) => {
-    rebuildIndex(item?: any): Promise<boolean>;
+    rebuildIndex(task?: any): Promise<boolean>;
     indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
     indexPendingData(): Promise<boolean>;
 };

package/dist/server/services/perform-indexing.js

@@ -1,17 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = ({ strapi }) => ({
-    async rebuildIndex(item = null) {
-        const helper = strapi.plugins['elasticsearch'].services.helper;
-        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
-        const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
-        const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
-        const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
-        const virtualCollectionsIndexer = strapi.plugins['elasticsearch'].services['virtualCollectionsIndexer'];
-        const virtualCollectionsRegistry = strapi.plugins['elasticsearch'].services['virtualCollectionsRegistry'];
+    async rebuildIndex(task = null) {
+        const pluginServices = strapi.plugins['elasticsearch'].services;
+        const helper = pluginServices.helper;
+        const esInterface = pluginServices.esInterface;
+        const scheduleIndexingService = pluginServices.scheduleIndexing;
+        const configureIndexingService = pluginServices.configureIndexing;
+        const logIndexingService = pluginServices.logIndexing;
+        const virtualCollectionsIndexer = pluginServices.virtualCollectionsIndexer;
+        const virtualCollectionsRegistry = pluginServices.virtualCollectionsRegistry;
+        let taskError = null;
         try {
             console.log('strapi-plugin-elasticsearch : Request to rebuild the index received.');
-            const fullIndexingInProgress = await scheduleIndexingService.getFullIndexingInProgress();
+            const fullIndexingInProgress = await scheduleIndexingService.getActiveFullIndexingTasks();
             if (fullIndexingInProgress.length > 0) {
                 const msg = `Indexing is already in progress - see tasks ${fullIndexingInProgress.map((t) => t.id)}. This request is ignored and marked as failed.`;
                 console.log('strapi-plugin-elasticsearch : ' + msg);
@@ -35,19 +37,19 @@ exports.default = ({ strapi }) => ({
             }
             //Step 2 : Index all the stuff on this new index
             console.log('strapi-plugin-elasticsearch : Starting to index all data into the new index.');
-            if (item == null) {
-                item = await scheduleIndexingService.addFullSiteIndexingTask();
+            if (task == null) {
+                task = await scheduleIndexingService.addFullSiteIndexingTask();
             }
-            if (item?.id) {
-                await scheduleIndexingService.markIndexingTaskInProgress(item.id);
+            if (task?.id) {
+                await scheduleIndexingService.markIndexingTaskInProgress(task.id);
                 let entitiesIndexed = 0;
                 for (let r = 0; r < cols.length; r++) {
                     entitiesIndexed += await this.indexCollection(cols[r], newIndexName);
                 }
                 // Indexing the virtual collections
-                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections. task id : ', item.id);
+                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections. task id : ', task.id);
                 const virtualEntriesIndexed = await virtualCollectionsIndexer.reindexAll(newIndexName);
-                await scheduleIndexingService.markIndexingTaskComplete(item.id);
+                await scheduleIndexingService.markIndexingTaskComplete(task.id);
                 console.log('strapi-plugin-elasticsearch : Indexing of data into the new index complete.');
                 //Step 4 : Move the alias to this new index
                 await esInterface.attachAliasToIndex(newIndexName);
@@ -64,14 +66,15 @@ exports.default = ({ strapi }) => ({
             }
         }
         catch (err) {
+            taskError = err.message || String(err);
             console.log('strapi-plugin-elasticsearch : searchController : An error was encountered while re-indexing.');
             console.log(err);
             await logIndexingService.recordIndexingFail(err);
             throw err;
         }
         finally {
-            if (item?.id) {
-                await scheduleIndexingService.markIndexingTaskComplete(item.id);
+            if (task?.id) {
+                await scheduleIndexingService.markIndexingTaskComplete(task.id, taskError);
             }
         }
     },

package/dist/server/services/schedule-indexing.d.ts

@@ -14,8 +14,8 @@ declare const _default: ({ strapi }: {
         recordId: any;
     }): Promise<void>;
     getItemsPendingToBeIndexed(): Promise<any>;
-    markIndexingTaskComplete(recId: any): Promise<void>;
+    markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
     markIndexingTaskInProgress(recId: any): Promise<void>;
-    getFullIndexingInProgress(): Promise<any>;
+    getActiveFullIndexingTasks(staleTaskThresholdSeconds?: number): Promise<any[]>;
 };
 export default _default;

package/dist/server/services/schedule-indexing.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const STUCK_THRESHOLD_SECONDS = 60 * 60; // 1 hour
 exports.default = ({ strapi }) => ({
     async addFullSiteIndexingTask() {
         const data = await strapi.entityService.create('plugin::elasticsearch.task', {
@@ -52,10 +53,12 @@ exports.default = ({ strapi }) => ({
         });
         return entries;
     },
-    async markIndexingTaskComplete(recId) {
+    async markIndexingTaskComplete(recId, error = null) {
+        const status = error ? 'failed' : 'done';
         const entries = await strapi.entityService.update('plugin::elasticsearch.task', recId, {
             data: {
-                indexing_status: 'done',
+                indexing_status: status,
+                error_message: error,
             },
         });
     },
@@ -66,13 +69,29 @@ exports.default = ({ strapi }) => ({
             },
         });
     },
-    async getFullIndexingInProgress() {
+    async getActiveFullIndexingTasks(staleTaskThresholdSeconds) {
+        if (!staleTaskThresholdSeconds) {
+            staleTaskThresholdSeconds = Number(strapi.config.get('plugin.elasticsearch').staleTaskThresholdSeconds) || STUCK_THRESHOLD_SECONDS;
+        }
+        staleTaskThresholdSeconds *= 1000;
         const entries = await strapi.entityService.findMany('plugin::elasticsearch.task', {
             filters: {
                 indexing_status: 'in-progress',
                 full_site_indexing: true,
             },
         });
-        return entries;
+        const now = Date.now();
+        const activeTasks = [];
+        for (const t of entries) {
+            const updatedAt = new Date(t.updatedAt || t.createdAt).getTime();
+            if (now - updatedAt > staleTaskThresholdSeconds) {
+                await strapi.plugins['elasticsearch'].services.logIndexing.recordIndexingFail(`Task ${t.id} was stuck in-progress for too long. Marking as failed.`);
+                await this.markIndexingTaskComplete(t.id, 'Stuck in-progress');
+            }
+            else {
+                activeTasks.push(t);
+            }
+        }
+        return activeTasks;
     },
 });

package/dist/server/services/virtual-collections-indexer.js

@@ -24,7 +24,9 @@ exports.default = ({ strapi }) => {
             try {
                 const results = await collection.extractByIds([itemId]);
                 if (!results || !Array.isArray(results) || results.length === 0) {
-                    strapi.log.warn(`No data extracted for ${collectionName} with ID ${itemId}`);
+                    // item does not exit - delete it from index
+                    await this.deleteItem(collectionName, itemId);
+                    strapi.log.debug(`Deleted virtual item: ${collectionName}:${itemId}`);
                     return null;
                 }
                 const itemData = results[0];
@@ -58,9 +60,11 @@ exports.default = ({ strapi }) => {
         async reindex(collection) {
             const collectionName = collection.collectionName;
             const privateIndexAlias = collection.indexAlias;
+            const pageSize = 100;
             const helper = getHelperService();
             let timestamp = Date.now();
             let indexName = '';
+            let errors = 0;
             try {
                 const esInterface = getElasticsearchService();
                 if (privateIndexAlias) {
@@ -75,7 +79,16 @@ exports.default = ({ strapi }) => {
                 let totalIndexed = 0;
                 const pageLimit = 10000;
                 while (page <= pageLimit) {
-                    const pageData = await collection.extractData(page);
+                    let pageData;
+                    try {
+                        pageData = await collection.extractData(page, pageSize);
+                    }
+                    catch (error) {
+                        strapi.log.error(`Error extracting data for page ${page} of ${collectionName}: ${error.message}`);
+                        errors += pageSize;
+                        page++;
+                        continue;
+                    }
                     strapi.log.debug(`Extracted ${pageData.length} items from ${collectionName} for page ${page}`);
                     if (!Array.isArray(pageData) || pageData.length === 0) {
                         break;
@@ -97,12 +110,18 @@ exports.default = ({ strapi }) => {
                         operations.push({ itemId, itemData });
                     }
                     if (operations.length > 0) {
-                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName)));
+                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName).catch((err) => {
+                            strapi.log.error(`Failed to index item ${op.itemId} in ${collectionName}: ${err}`);
+                            errors++;
+                        })));
                     }
                     totalIndexed += pageData.length;
                     prevPageData = pageData;
                     page++;
                 }
+                if (errors > 0) {
+                    throw new Error(`Failed to index ${errors} of ${totalIndexed} items for virtual collection ${collectionName}. Errors were logged. Alias was not updated. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                }
                 strapi.log.info(`Reindexed ${totalIndexed} items for virtual collection: ${collectionName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}. now updating alias.`);
                 if (privateIndexAlias) {
                     timestamp = Date.now();
@@ -138,14 +157,7 @@ exports.default = ({ strapi }) => {
                 const idsToReindex = await trigger.getIdsToReindex(result);
                 // Reindex each item
                 for (const id of idsToReindex) {
-                    const isDelete = event.action?.toLowerCase()?.includes('delete') && trigger.alsoTriggerDelete && id === result.id;
-                    if (isDelete) {
-                        //delete the item from the index, if the item being delete is the one being reindexed
-                        await this.deleteItem(collection.collectionName, id);
-                    }
-                    else {
-                        await this.indexItem(collection.collectionName, id);
-                    }
+                    await this.indexItem(collection.collectionName, id);
                 }
             }
         },

package/dist/server/services/virtual-collections-registry.js

@@ -47,7 +47,6 @@ const configSchema = yup.object({
         .of(yup.object({
         collection: yup.string().required(),
         getIdsToReindex: isFunction().required(),
-        alsoTriggerDelete: yup.boolean().default(false),
     }))
         .default([]),
     mappings: yup.object().optional(),

package/dist/server/types/virtual-collections.type.d.ts

@@ -45,11 +45,6 @@ export type VirtualCollectionConfig = {
          * @returns ids of the items to be reindexed.
          */
         getIdsToReindex: GetIdsToIndexFunction;
-        /**
-         * if true, and the trigger is a delete event, the item of the virtual collection will be deleted as well if the id returned from getIdsToReindex match.
-         * defaults to false.
-         */
-        alsoTriggerDelete?: boolean;
     }>;
     /**
      * Optional schema to be sent to Elasticsearch when creating the index.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.2.9",
+  "version": "0.3.1",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {
@@ -54,7 +54,7 @@
     }
   ],
   "engines": {
-    "node": ">=16.0.0 <=20.x.x",
+    "node": ">=16.0.0",
     "npm": ">=6.0.0"
   },
   "license": "MIT",
@@ -64,4 +64,4 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.8.3"
   }
-}
+}