@vegan-friendly/strapi-plugin-elasticsearch 0.2.9 → 0.3.0

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-    "version": "0.2.9",
+    "version": "0.3.0",
     "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
     "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
     "strapi": {
@@ -54,7 +54,7 @@
         }
     ],
     "engines": {
-        "node": ">=16.0.0 <=20.x.x",
+        "node": ">=16.0.0",
         "npm": ">=6.0.0"
     },
     "license": "MIT",

package/dist/server/content-types/index.d.ts

@@ -34,6 +34,10 @@ declare const _default: {
                     required: boolean;
                     default: string;
                 };
+                error_message: {
+                    type: string;
+                    required: boolean;
+                };
                 full_site_indexing: {
                     type: string;
                 };

package/dist/server/content-types/tasks.d.ts

@@ -32,6 +32,10 @@ declare const _default: {
             required: boolean;
             default: string;
         };
+        error_message: {
+            type: string;
+            required: boolean;
+        };
         full_site_indexing: {
             type: string;
         };

package/dist/server/content-types/tasks.js

@@ -30,10 +30,14 @@ exports.default = {
         },
         indexing_status: {
             type: 'enumeration',
-            enum: ['to-be-done', 'in-progress', 'done'],
+            enum: ['to-be-done', 'in-progress', 'done', 'failed'],
             required: true,
             default: 'to-be-done',
         },
+        error_message: {
+            type: 'string',
+            required: false,
+        },
         full_site_indexing: {
             type: 'boolean',
         },

package/dist/server/index.d.ts

@@ -142,7 +142,7 @@ declare const _default: {
                 recordId: any;
             }): Promise<void>;
             getItemsPendingToBeIndexed(): Promise<any>;
-            markIndexingTaskComplete(recId: any): Promise<void>;
+            markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
             markIndexingTaskInProgress(recId: any): Promise<void>;
             getFullIndexingInProgress(): Promise<any>;
         };
@@ -152,7 +152,7 @@ declare const _default: {
         indexer: ({ strapi }: {
             strapi: any;
         }) => {
-            rebuildIndex(item?: any): Promise<boolean>;
+            rebuildIndex(task?: any): Promise<boolean>;
             indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
             indexPendingData(): Promise<boolean>;
         };
@@ -215,6 +215,10 @@ declare const _default: {
                         required: boolean;
                         default: string;
                     };
+                    error_message: {
+                        type: string;
+                        required: boolean;
+                    };
                     full_site_indexing: {
                         type: string;
                     };

package/dist/server/services/index.d.ts

@@ -38,7 +38,7 @@ declare const _default: {
             recordId: any;
         }): Promise<void>;
         getItemsPendingToBeIndexed(): Promise<any>;
-        markIndexingTaskComplete(recId: any): Promise<void>;
+        markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
         markIndexingTaskInProgress(recId: any): Promise<void>;
         getFullIndexingInProgress(): Promise<any>;
     };
@@ -48,7 +48,7 @@ declare const _default: {
     indexer: ({ strapi }: {
         strapi: any;
     }) => {
-        rebuildIndex(item?: any): Promise<boolean>;
+        rebuildIndex(task?: any): Promise<boolean>;
         indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
         indexPendingData(): Promise<boolean>;
     };

package/dist/server/services/perform-indexing.d.ts

@@ -1,7 +1,7 @@
 declare const _default: ({ strapi }: {
     strapi: any;
 }) => {
-    rebuildIndex(item?: any): Promise<boolean>;
+    rebuildIndex(task?: any): Promise<boolean>;
     indexCollection(collectionName: any, indexName?: string | null): Promise<number>;
     indexPendingData(): Promise<boolean>;
 };

package/dist/server/services/perform-indexing.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = ({ strapi }) => ({
-    async rebuildIndex(item = null) {
+    async rebuildIndex(task = null) {
         const helper = strapi.plugins['elasticsearch'].services.helper;
         const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
         const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
@@ -9,6 +9,7 @@ exports.default = ({ strapi }) => ({
         const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
         const virtualCollectionsIndexer = strapi.plugins['elasticsearch'].services['virtualCollectionsIndexer'];
         const virtualCollectionsRegistry = strapi.plugins['elasticsearch'].services['virtualCollectionsRegistry'];
+        let taskError = null;
         try {
             console.log('strapi-plugin-elasticsearch : Request to rebuild the index received.');
             const fullIndexingInProgress = await scheduleIndexingService.getFullIndexingInProgress();
@@ -35,19 +36,19 @@ exports.default = ({ strapi }) => ({
             }
             //Step 2 : Index all the stuff on this new index
             console.log('strapi-plugin-elasticsearch : Starting to index all data into the new index.');
-            if (item == null) {
-                item = await scheduleIndexingService.addFullSiteIndexingTask();
+            if (task == null) {
+                task = await scheduleIndexingService.addFullSiteIndexingTask();
             }
-            if (item?.id) {
-                await scheduleIndexingService.markIndexingTaskInProgress(item.id);
+            if (task?.id) {
+                await scheduleIndexingService.markIndexingTaskInProgress(task.id);
                 let entitiesIndexed = 0;
                 for (let r = 0; r < cols.length; r++) {
                     entitiesIndexed += await this.indexCollection(cols[r], newIndexName);
                 }
                 // Indexing the virtual collections
-                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections. task id : ', item.id);
+                console.log('strapi-plugin-elasticsearch : Starting to index virtual collections. task id : ', task.id);
                 const virtualEntriesIndexed = await virtualCollectionsIndexer.reindexAll(newIndexName);
-                await scheduleIndexingService.markIndexingTaskComplete(item.id);
+                await scheduleIndexingService.markIndexingTaskComplete(task.id);
                 console.log('strapi-plugin-elasticsearch : Indexing of data into the new index complete.');
                 //Step 4 : Move the alias to this new index
                 await esInterface.attachAliasToIndex(newIndexName);
@@ -64,14 +65,15 @@ exports.default = ({ strapi }) => ({
             }
         }
         catch (err) {
+            taskError = err.message || String(err);
             console.log('strapi-plugin-elasticsearch : searchController : An error was encountered while re-indexing.');
             console.log(err);
             await logIndexingService.recordIndexingFail(err);
             throw err;
         }
         finally {
-            if (item?.id) {
-                await scheduleIndexingService.markIndexingTaskComplete(item.id);
+            if (task?.id) {
+                await scheduleIndexingService.markIndexingTaskComplete(task.id, taskError);
             }
         }
     },

package/dist/server/services/schedule-indexing.d.ts

@@ -14,7 +14,7 @@ declare const _default: ({ strapi }: {
         recordId: any;
     }): Promise<void>;
     getItemsPendingToBeIndexed(): Promise<any>;
-    markIndexingTaskComplete(recId: any): Promise<void>;
+    markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
     markIndexingTaskInProgress(recId: any): Promise<void>;
     getFullIndexingInProgress(): Promise<any>;
 };

package/dist/server/services/schedule-indexing.js

@@ -52,10 +52,12 @@ exports.default = ({ strapi }) => ({
         });
         return entries;
     },
-    async markIndexingTaskComplete(recId) {
+    async markIndexingTaskComplete(recId, error = null) {
+        const status = error ? 'failed' : 'done';
         const entries = await strapi.entityService.update('plugin::elasticsearch.task', recId, {
             data: {
-                indexing_status: 'done',
+                indexing_status: status,
+                error_message: error,
             },
         });
     },

package/dist/server/services/virtual-collections-indexer.js

@@ -24,7 +24,9 @@ exports.default = ({ strapi }) => {
             try {
                 const results = await collection.extractByIds([itemId]);
                 if (!results || !Array.isArray(results) || results.length === 0) {
-                    strapi.log.warn(`No data extracted for ${collectionName} with ID ${itemId}`);
+                    // item does not exit - delete it from index
+                    await this.deleteItem(collectionName, itemId);
+                    strapi.log.debug(`Deleted virtual item: ${collectionName}:${itemId}`);
                     return null;
                 }
                 const itemData = results[0];
@@ -58,9 +60,11 @@ exports.default = ({ strapi }) => {
         async reindex(collection) {
             const collectionName = collection.collectionName;
             const privateIndexAlias = collection.indexAlias;
+            const pageSize = 100;
             const helper = getHelperService();
             let timestamp = Date.now();
             let indexName = '';
+            let errors = 0;
             try {
                 const esInterface = getElasticsearchService();
                 if (privateIndexAlias) {
@@ -75,7 +79,16 @@ exports.default = ({ strapi }) => {
                 let totalIndexed = 0;
                 const pageLimit = 10000;
                 while (page <= pageLimit) {
-                    const pageData = await collection.extractData(page);
+                    let pageData;
+                    try {
+                        pageData = await collection.extractData(page, pageSize);
+                    }
+                    catch (error) {
+                        strapi.log.error(`Error extracting data for page ${page} of ${collectionName}: ${error.message}`);
+                        errors += pageSize;
+                        page++;
+                        continue;
+                    }
                     strapi.log.debug(`Extracted ${pageData.length} items from ${collectionName} for page ${page}`);
                     if (!Array.isArray(pageData) || pageData.length === 0) {
                         break;
@@ -97,12 +110,18 @@ exports.default = ({ strapi }) => {
                         operations.push({ itemId, itemData });
                     }
                     if (operations.length > 0) {
-                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName)));
+                        await Promise.all(operations.map((op) => esInterface.indexDataToSpecificIndex(op, indexName).catch((err) => {
+                            strapi.log.error(`Failed to index item ${op.itemId} in ${collectionName}: ${err}`);
+                            errors++;
+                        })));
                     }
                     totalIndexed += pageData.length;
                     prevPageData = pageData;
                     page++;
                 }
+                if (errors > 0) {
+                    throw new Error(`Failed to index ${errors} of ${totalIndexed} items for virtual collection ${collectionName}. Errors were logged. Alias was not updated. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}`);
+                }
                 strapi.log.info(`Reindexed ${totalIndexed} items for virtual collection: ${collectionName}. took ${(0, humanize_duration_1.default)(Date.now() - timestamp)}. now updating alias.`);
                 if (privateIndexAlias) {
                     timestamp = Date.now();
@@ -138,14 +157,7 @@ exports.default = ({ strapi }) => {
                 const idsToReindex = await trigger.getIdsToReindex(result);
                 // Reindex each item
                 for (const id of idsToReindex) {
-                    const isDelete = event.action?.toLowerCase()?.includes('delete') && trigger.alsoTriggerDelete && id === result.id;
-                    if (isDelete) {
-                        //delete the item from the index, if the item being delete is the one being reindexed
-                        await this.deleteItem(collection.collectionName, id);
-                    }
-                    else {
-                        await this.indexItem(collection.collectionName, id);
-                    }
+                    await this.indexItem(collection.collectionName, id);
                 }
             }
         },

package/dist/server/services/virtual-collections-registry.js

@@ -47,7 +47,6 @@ const configSchema = yup.object({
         .of(yup.object({
         collection: yup.string().required(),
         getIdsToReindex: isFunction().required(),
-        alsoTriggerDelete: yup.boolean().default(false),
     }))
         .default([]),
     mappings: yup.object().optional(),

package/dist/server/types/virtual-collections.type.d.ts

@@ -45,11 +45,6 @@ export type VirtualCollectionConfig = {
          * @returns ids of the items to be reindexed.
          */
         getIdsToReindex: GetIdsToIndexFunction;
-        /**
-         * if true, and the trigger is a delete event, the item of the virtual collection will be deleted as well if the id returned from getIdsToReindex match.
-         * defaults to false.
-         */
-        alsoTriggerDelete?: boolean;
     }>;
     /**
      * Optional schema to be sent to Elasticsearch when creating the index.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.2.9",
+  "version": "0.3.0",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {
@@ -54,7 +54,7 @@
     }
   ],
   "engines": {
-    "node": ">=16.0.0 <=20.x.x",
+    "node": ">=16.0.0",
     "npm": ">=6.0.0"
   },
   "license": "MIT",
@@ -64,4 +64,4 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.8.3"
   }
-}
+}