@vegan-friendly/strapi-plugin-elasticsearch 0.3.0 → 0.3.2

package/README.md

@@ -257,6 +257,111 @@ module.exports = (plugin) => {
 - This will create a new route `/api/elasticsearch/enhanced-search` being served by the function defined above.
 - You can add / modify the routes and controllers as necessary.
 
+## Virtual Collections
+
+**Virtual Collections** allow you to index and search data that does not directly map to a single Strapi collection type. This is useful for aggregating, transforming, or combining data from multiple sources before indexing it in Elasticsearch.
+
+### What is a Virtual Collection?
+
+A virtual collection is a logical grouping of data that you define, which can be indexed into Elasticsearch as if it were a regular Strapi collection. You control how the data is extracted, transformed, and indexed.
+
+For example -  
+Let's say that you have two collections defined in Strapi:
+
+- Restaurant
+- Chain
+
+* Each chain is related to one or more restaurants. Some restaurants don't have a chain.
+
+Now let's say you want to index the restaurants, but in each restaurant you'd like to include some information about the restaurant, and you also want to merge some fields (e.g: if a Restaurant doesn't have a `description` - it inherits it from its chain ).  
+To do that, you need to create a virtual-collection in this plugin configuration, based on your existing restaurant collection.
+
+### How to Register a Virtual Collection
+
+To register a virtual collection, you need to provide a configuration object that implements the `VirtualCollectionConfig` interface. This is typically done in your plugin or project code.
+
+**Example:**
+
+```typescript
+// src/extensions/elasticsearch/virtual-collections/my-virtual-collection.ts
+
+import { VirtualCollectionConfig } from 'strapi-plugin-elasticsearch/server/types/virtual-collections.type';
+
+const myVirtualCollection: VirtualCollectionConfig = {
+  collectionName: 'virtual::my-virtual-collection',
+  indexAlias: 'my_virtual_collection_index',
+  extractData: async (page, pageSize) => {
+    // Fetch and return an array of entities for the given page
+    // Example: aggregate data from multiple collections
+    return [];
+  },
+  extractByIds: async (ids) => {
+    // Fetch and return entities by their IDs
+    return [];
+  },
+  getIndexItemId: (itemId, collectionName) => `${collectionName}::${itemId}`,
+  triggers: [
+    {
+      collection: 'api::some-collection.some-collection',
+      getIdsToReindex: (event) => {
+        // Return an array of virtual collection item IDs to reindex
+        return [event.result.id];
+      },
+    },
+  ],
+  mappings: {
+    // Optional: Elasticsearch mappings for this collection
+  },
+};
+
+export default myVirtualCollection;
+```
+
+### Key Properties
+
+- **collectionName**: Unique name for your virtual collection.
+- **indexAlias**: (Optional) Alias for the Elasticsearch index.
+- **extractData(page, pageSize)**: Function to fetch paginated data for indexing.
+- **extractByIds(ids)**: Function to fetch specific items by ID.
+- **getIndexItemId(itemId, collectionName)**: (Optional) Function to generate unique Elasticsearch document IDs.
+- **triggers**: Array of triggers that specify which Strapi collections should cause this virtual collection to reindex.
+- **mappings**: (Optional) Elasticsearch mappings for the index.
+
+### How Triggers Work
+
+Each trigger listens to changes on a specified Strapi collection. When a change occurs, the `getIdsToReindex` function is called with the event data, and should return the IDs of the virtual collection items that need to be reindexed.  
+These items are then reindexed - they are fetched using `extractByIds` and are sent to ElasticSearch. If an item cannot be found using `extractByIds` - it is deleted from ElasticSearch.
+
+### Registering the Virtual Collection
+
+Register your virtual collection by specifying the `virtualCollections` property in this plugin's configuration, in `config/plugins.js`.
+
+**Example:**
+
+```javascript
+// config/plugins.js
+
+import myVirtualCollection from './virtual-collections/my-virtual-collection';
+
+module.exports = async ({ env }) => {
+  return {
+    elasticsearch :{
+      enabled: true,
+      config: {
+        ...
+        virtualCollections: [
+          myVirtualCollection,
+        ]
+      }
+    }
+  }
+};
+```
+
+### Indexing and Searching
+
+Once registered, your virtual collection will be indexed according to the triggers and can be searched like any other indexed collection in Elasticsearch.
+
 ## Bugs
 For any bugs, please create an issue [here](https://github.com/geeky-biz/strapi-plugin-elasticsearch/issues).
 

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-    "version": "0.3.0",
+    "version": "0.3.2",
     "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
     "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
     "strapi": {

package/dist/server/bootstrap.js

@@ -146,16 +146,19 @@ exports.default = async ({ strapi }) => {
             strapi.db.lifecycles.subscribe({
                 models: [collectionUID],
                 afterCreate: async (event) => {
-                    await virtualCollectionIndexer.handleTriggerEvent(event);
+                    virtualCollectionIndexer.handleTriggerEvent(event);
                 },
                 afterUpdate: async (event) => {
-                    await virtualCollectionIndexer.handleTriggerEvent(event);
+                    virtualCollectionIndexer.handleTriggerEvent(event);
                 },
                 afterDelete: async (event) => {
-                    await virtualCollectionIndexer.handleTriggerEvent(event);
+                    virtualCollectionIndexer.handleTriggerEvent(event);
                 },
             });
         });
+        // clean up old indexing tasks, as server is booting.
+        // allow 60 seconds, in case strapi is being run in cluster mode and this is the second instance
+        await scheduleIndexingService.getActiveFullIndexingTasks(60);
         configureIndexingService.markInitialized();
     }
     catch (err) {

package/dist/server/index.d.ts

@@ -144,7 +144,7 @@ declare const _default: {
             getItemsPendingToBeIndexed(): Promise<any>;
             markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
             markIndexingTaskInProgress(recId: any): Promise<void>;
-            getFullIndexingInProgress(): Promise<any>;
+            getActiveFullIndexingTasks(staleTaskThresholdSeconds?: number): Promise<any[]>;
         };
         esInterface: ({ strapi }: {
             strapi: any;

package/dist/server/services/index.d.ts

@@ -40,7 +40,7 @@ declare const _default: {
         getItemsPendingToBeIndexed(): Promise<any>;
         markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
         markIndexingTaskInProgress(recId: any): Promise<void>;
-        getFullIndexingInProgress(): Promise<any>;
+        getActiveFullIndexingTasks(staleTaskThresholdSeconds?: number): Promise<any[]>;
     };
     esInterface: ({ strapi }: {
         strapi: any;

package/dist/server/services/perform-indexing.js

@@ -2,17 +2,18 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = ({ strapi }) => ({
     async rebuildIndex(task = null) {
-        const helper = strapi.plugins['elasticsearch'].services.helper;
-        const esInterface = strapi.plugins['elasticsearch'].services.esInterface;
-        const scheduleIndexingService = strapi.plugins['elasticsearch'].services.scheduleIndexing;
-        const configureIndexingService = strapi.plugins['elasticsearch'].services.configureIndexing;
-        const logIndexingService = strapi.plugins['elasticsearch'].services.logIndexing;
-        const virtualCollectionsIndexer = strapi.plugins['elasticsearch'].services['virtualCollectionsIndexer'];
-        const virtualCollectionsRegistry = strapi.plugins['elasticsearch'].services['virtualCollectionsRegistry'];
+        const pluginServices = strapi.plugins['elasticsearch'].services;
+        const helper = pluginServices.helper;
+        const esInterface = pluginServices.esInterface;
+        const scheduleIndexingService = pluginServices.scheduleIndexing;
+        const configureIndexingService = pluginServices.configureIndexing;
+        const logIndexingService = pluginServices.logIndexing;
+        const virtualCollectionsIndexer = pluginServices.virtualCollectionsIndexer;
+        const virtualCollectionsRegistry = pluginServices.virtualCollectionsRegistry;
         let taskError = null;
         try {
             console.log('strapi-plugin-elasticsearch : Request to rebuild the index received.');
-            const fullIndexingInProgress = await scheduleIndexingService.getFullIndexingInProgress();
+            const fullIndexingInProgress = await scheduleIndexingService.getActiveFullIndexingTasks();
             if (fullIndexingInProgress.length > 0) {
                 const msg = `Indexing is already in progress - see tasks ${fullIndexingInProgress.map((t) => t.id)}. This request is ignored and marked as failed.`;
                 console.log('strapi-plugin-elasticsearch : ' + msg);

package/dist/server/services/schedule-indexing.d.ts

@@ -16,6 +16,6 @@ declare const _default: ({ strapi }: {
     getItemsPendingToBeIndexed(): Promise<any>;
     markIndexingTaskComplete(recId: any, error?: string | null): Promise<void>;
     markIndexingTaskInProgress(recId: any): Promise<void>;
-    getFullIndexingInProgress(): Promise<any>;
+    getActiveFullIndexingTasks(staleTaskThresholdSeconds?: number): Promise<any[]>;
 };
 export default _default;

package/dist/server/services/schedule-indexing.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const STUCK_THRESHOLD_SECONDS = 60 * 60; // 1 hour
 exports.default = ({ strapi }) => ({
     async addFullSiteIndexingTask() {
         const data = await strapi.entityService.create('plugin::elasticsearch.task', {
@@ -68,13 +69,29 @@ exports.default = ({ strapi }) => ({
             },
         });
     },
-    async getFullIndexingInProgress() {
+    async getActiveFullIndexingTasks(staleTaskThresholdSeconds) {
+        if (!staleTaskThresholdSeconds) {
+            staleTaskThresholdSeconds = Number(strapi.config.get('plugin.elasticsearch').staleTaskThresholdSeconds) || STUCK_THRESHOLD_SECONDS;
+        }
+        staleTaskThresholdSeconds *= 1000;
         const entries = await strapi.entityService.findMany('plugin::elasticsearch.task', {
             filters: {
                 indexing_status: 'in-progress',
                 full_site_indexing: true,
             },
         });
-        return entries;
+        const now = Date.now();
+        const activeTasks = [];
+        for (const t of entries) {
+            const updatedAt = new Date(t.updatedAt || t.createdAt).getTime();
+            if (now - updatedAt > staleTaskThresholdSeconds) {
+                await strapi.plugins['elasticsearch'].services.logIndexing.recordIndexingFail(`Task ${t.id} was stuck in-progress for too long. Marking as failed.`);
+                await this.markIndexingTaskComplete(t.id, 'Stuck in-progress');
+            }
+            else {
+                activeTasks.push(t);
+            }
+        }
+        return activeTasks;
     },
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {