@vegan-friendly/strapi-plugin-elasticsearch 0.0.11-alpha.7 → 0.1.0-alpha.0

package/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "endOfLine": "lf",
+  "tabWidth": 2,
+  "printWidth": 100,
+  "singleQuote": true,
+  "trailingComma": "es5"
+}

package/.vscode/settings.json

@@ -1,3 +1,24 @@
 {
   "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "[javascript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[javascriptreact]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[typescriptreact]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[jsonc]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[json]": {
+    //redundant, but just in case
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "editor.formatOnSaveMode": "modifications",
+  "editor.formatOnSave": true
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegan-friendly/strapi-plugin-elasticsearch",
-  "version": "0.0.11-alpha.7",
+  "version": "0.1.0-alpha.0",
   "description": "A Strapi plugin to enable using Elasticsearch with Strapi CMS.",
   "homepage": "https://github.com/vegan-friendly/strapi-plugin-elasticsearch",
   "strapi": {

package/server/services/index.js

@@ -7,6 +7,7 @@ const indexer = require('./perform-indexing');
 const logIndexing = require('./log-indexing');
 const helper = require('./helper');
 const transformContent = require('./transform-content');
+const virtualCollectionsRegistry = require('./virtual-collections-registry');
 
 module.exports = {
   configureIndexing,
@@ -15,5 +16,6 @@ module.exports = {
   indexer,
   logIndexing,
   helper,
-  transformContent
+  transformContent,
+  virtualCollectionsRegistry,
 };

package/server/services/virtual-collections-registry.js

@@ -0,0 +1,445 @@
+/**
+ * Service to manage virtual collections for Elasticsearch indexing
+ */
+module.exports = ({ strapi }) => {
+  // Registry to hold all virtual collection definitions
+  const virtualCollections = new Map();
+
+  return {
+    /**
+     * Initialize the registry from the plugin configuration
+     */
+    initialize() {
+      // Get the plugin configuration
+      const pluginConfig = strapi.config.get('plugin.elasticsearch') || {};
+      const virtualCollectionsConfig = pluginConfig.virtualCollections || [];
+
+      // Register each virtual collection from the config
+      virtualCollectionsConfig.forEach((config) => {
+        this.register(config);
+      });
+
+      strapi.log.info(
+        `Initialized ${virtualCollections.size} virtual collections for Elasticsearch indexing`
+      );
+    },
+
+    /**
+     * Register a new virtual collection for indexing
+     * @param {Object} config - Configuration for the virtual collection
+     * @param {string} config.indexName - Name of the index in Elasticsearch
+     * @param {string} config.collectionName - Name identifier for the virtual collection
+     * @param {Function} config.extractData - Function that returns data to be indexed (with pagination)
+     * @param {Array<Object>} config.triggers - Array of collection triggers that should cause reindexing
+     * @param {Function} config.mapToIndex - Function that maps extracted data to Elasticsearch document
+     */
+    register(config) {
+      if (!config.indexName || typeof config.indexName !== 'string') {
+        throw new Error('Virtual collection must have an indexName');
+      }
+
+      if (!config.collectionName || typeof config.collectionName !== 'string') {
+        throw new Error('Virtual collection must have a collectionName');
+      }
+
+      if (typeof config.extractData !== 'function') {
+        throw new Error('Virtual collection must have an extractData function');
+      }
+
+      if (!Array.isArray(config.triggers)) {
+        throw new Error('Virtual collection must have triggers defined');
+      }
+
+      if (typeof config.mapToIndex !== 'function') {
+        throw new Error('Virtual collection must have a mapToIndex function');
+      }
+
+      virtualCollections.set(config.collectionName, config);
+      strapi.log.info(
+        `Registered virtual collection for Elasticsearch: ${config.collectionName} -> ${config.indexName}`
+      );
+
+      return this;
+    },
+
+    /**
+     * Get all registered virtual collections
+     */
+    getAll() {
+      return Array.from(virtualCollections.values());
+    },
+
+    /**
+     * Get a specific virtual collection by name
+     */
+    get(collectionName) {
+      return virtualCollections.get(collectionName);
+    },
+
+    /**
+     * Find virtual collections that should be triggered when a specific collection changes
+     */
+    findTriggersByCollection(collectionUID) {
+      const results = [];
+
+      virtualCollections.forEach((config) => {
+        const hasTrigger = config.triggers.some((trigger) => trigger.collection === collectionUID);
+
+        if (hasTrigger) {
+          results.push(config);
+        }
+      });
+
+      return results;
+    },
+  };
+};
+
+// path: ./src/extensions/strapi-plugin-elasticsearch/services/virtual-collections-indexer.js
+
+/**
+ * Service to handle indexing of virtual collections
+ */
+module.exports = ({ strapi }) => {
+  const getElasticsearchService = () => strapi.plugin('elasticsearch').service('elasticsearch');
+  const getRegistryService = () =>
+    strapi.service('plugin::elasticsearch.virtual-collections-registry');
+
+  return {
+    /**
+     * Initialize indexes for all registered virtual collections
+     */
+    async initializeIndexes() {
+      const registry = getRegistryService();
+      const collections = registry.getAll();
+
+      for (const collection of collections) {
+        await this.createIndexIfNotExists(collection.indexName);
+      }
+    },
+
+    /**
+     * Create an Elasticsearch index if it doesn't exist
+     */
+    async createIndexIfNotExists(indexName) {
+      const esService = getElasticsearchService();
+      const indexExists = await esService.indices.exists({ index: indexName });
+
+      if (!indexExists) {
+        await esService.indices.create({
+          index: indexName,
+          body: {
+            settings: {
+              analysis: {
+                analyzer: {
+                  default: {
+                    type: 'standard',
+                  },
+                },
+              },
+            },
+          },
+        });
+
+        strapi.log.info(`Created Elasticsearch index: ${indexName}`);
+      }
+    },
+
+    /**
+     * Index a single item from a virtual collection
+     */
+    async indexItem(collectionName, itemId) {
+      const registry = getRegistryService();
+      const collection = registry.get(collectionName);
+
+      if (!collection) {
+        throw new Error(`Virtual collection not found: ${collectionName}`);
+      }
+
+      try {
+        // Extract data using the collection's extractData
+        // For a single item, we'll pass the ID as a filter parameter
+        const results = await collection.extractData(0, { id: itemId });
+
+        if (!results || !Array.isArray(results) || results.length === 0) {
+          strapi.log.warn(`No data extracted for ${collectionName} with ID ${itemId}`);
+          return null;
+        }
+
+        const data = results[0];
+
+        // Map the data to the index format
+        const indexData = collection.mapToIndex(data);
+
+        // Index the data
+        const esService = getElasticsearchService();
+        await esService.index({
+          index: collection.indexName,
+          id: itemId,
+          body: indexData,
+          refresh: true,
+        });
+
+        strapi.log.debug(`Indexed virtual item: ${collectionName}:${itemId}`);
+        return indexData;
+      } catch (error) {
+        strapi.log.error(`Error indexing ${collectionName}:${itemId}: ${error.message}`);
+        throw error;
+      }
+    },
+
+    /**
+     * Reindex all items in a virtual collection
+     */
+    async reindexAll(collectionName) {
+      const registry = getRegistryService();
+      const collection = registry.get(collectionName);
+
+      if (!collection) {
+        throw new Error(`Virtual collection not found: ${collectionName}`);
+      }
+
+      try {
+        // Create a new index with a timestamp
+        const timestamp = Date.now();
+        const tempIndexName = `${collection.indexName}_${timestamp}`;
+
+        // Create the temporary index
+        const esService = getElasticsearchService();
+        await esService.indices.create({
+          index: tempIndexName,
+          body: {
+            settings: {
+              analysis: {
+                analyzer: {
+                  default: {
+                    type: 'standard',
+                  },
+                },
+              },
+            },
+          },
+        });
+
+        // Pagination variables
+        let page = 0;
+        let hasMoreData = true;
+        let totalIndexed = 0;
+
+        // Process data in batches
+        while (hasMoreData) {
+          // Extract data for current page
+          const pageData = await collection.extractData(page);
+
+          if (!Array.isArray(pageData) || pageData.length === 0) {
+            hasMoreData = false;
+            break;
+          }
+
+          // Bulk index items in this page
+          const operations = [];
+
+          for (const item of pageData) {
+            const indexData = collection.mapToIndex(item);
+            operations.push({
+              index: {
+                _index: tempIndexName,
+                _id: item.id,
+              },
+            });
+            operations.push(indexData);
+          }
+
+          if (operations.length > 0) {
+            await esService.bulk({ body: operations, refresh: true });
+          }
+
+          totalIndexed += pageData.length;
+          page++;
+        }
+
+        // Swap the indices
+        const oldIndex = collection.indexName;
+        const indexExists = await esService.indices.exists({ index: oldIndex });
+
+        if (indexExists) {
+          // Create or update alias
+          const aliasExists = await esService.indices.existsAlias({
+            name: collection.indexName,
+          });
+
+          if (aliasExists) {
+            // Update existing alias
+            await esService.indices.updateAliases({
+              body: {
+                actions: [
+                  { remove: { index: '_all', alias: collection.indexName } },
+                  { add: { index: tempIndexName, alias: collection.indexName } },
+                ],
+              },
+            });
+          } else {
+            // Create new alias
+            await esService.indices.putAlias({
+              index: tempIndexName,
+              name: collection.indexName,
+            });
+          }
+
+          // Delete old indices with this prefix
+          const { body: indices } = await esService.indices.get({
+            index: `${collection.indexName}_*`,
+          });
+
+          for (const indexName in indices) {
+            if (indexName !== tempIndexName) {
+              await esService.indices.delete({ index: indexName });
+            }
+          }
+        } else {
+          // Just add the alias if the original index doesn't exist
+          await esService.indices.putAlias({
+            index: tempIndexName,
+            name: collection.indexName,
+          });
+        }
+
+        strapi.log.info(
+          `Reindexed ${totalIndexed} items for virtual collection: ${collectionName}`
+        );
+        return totalIndexed;
+      } catch (error) {
+        strapi.log.error(`Error reindexing ${collectionName}: ${error.message}`);
+        throw error;
+      }
+    },
+
+    /**
+     * Handle a trigger event from a collection
+     */
+    async handleTriggerEvent(event) {
+      const { model, result } = event;
+      const registry = getRegistryService();
+
+      // Find virtual collections that should be triggered by this model
+      const affectedCollections = registry.findTriggersByCollection(model);
+
+      for (const collection of affectedCollections) {
+        // Find the specific trigger for this collection
+        const trigger = collection.triggers.find((t) => t.collection === model);
+
+        if (trigger && trigger.getIdsToReindex) {
+          // Get IDs that need to be reindexed
+          const idsToReindex = await trigger.getIdsToReindex(result);
+
+          // Reindex each item
+          for (const id of idsToReindex) {
+            await this.indexItem(collection.collectionName, id);
+          }
+        }
+      }
+    },
+
+    /**
+     * Delete an item from a virtual collection index
+     */
+    async deleteItem(collectionName, itemId) {
+      const registry = getRegistryService();
+      const collection = registry.get(collectionName);
+
+      if (!collection) {
+        throw new Error(`Virtual collection not found: ${collectionName}`);
+      }
+
+      try {
+        const esService = getElasticsearchService();
+        await esService.delete({
+          index: collection.indexName,
+          id: itemId,
+          refresh: true,
+        });
+
+        strapi.log.debug(`Deleted indexed item: ${collectionName}:${itemId}`);
+        return true;
+      } catch (error) {
+        if (error.meta && error.meta.statusCode === 404) {
+          // Item not found - that's fine
+          return false;
+        }
+        strapi.log.error(`Error deleting ${collectionName}:${itemId}: ${error.message}`);
+        throw error;
+      }
+    },
+  };
+};
+
+// path: ./src/extensions/strapi-plugin-elasticsearch/strapi-server.js
+
+module.exports = (plugin) => {
+  // Preserve original services, controllers, etc.
+  const originalServices = plugin.services || {};
+
+  // Add our new services
+  plugin.services = {
+    ...originalServices,
+    'virtual-collections-registry': require('./services/virtual-collections-registry'),
+    'virtual-collections-indexer': require('./services/virtual-collections-indexer'),
+  };
+
+  // Extend the bootstrap function
+  const originalBootstrap = plugin.bootstrap;
+  plugin.bootstrap = async ({ strapi }) => {
+    // Call the original bootstrap if it exists
+    if (originalBootstrap) {
+      await originalBootstrap({ strapi });
+    }
+
+    // Initialize the registry from config
+    strapi.service('plugin::elasticsearch.virtual-collections-registry').initialize();
+
+    // Setup lifecycle hooks
+    const registry = strapi.service('plugin::elasticsearch.virtual-collections-registry');
+    const virtualCollections = registry.getAll();
+
+    // Create a set of all collections that need hooks
+    const collectionsToHook = new Set();
+
+    virtualCollections.forEach((collection) => {
+      collection.triggers.forEach((trigger) => {
+        collectionsToHook.add(trigger.collection);
+      });
+    });
+
+    // Setup hooks for each collection
+    collectionsToHook.forEach((collectionUID) => {
+      strapi.log.info(`Setting up Elasticsearch lifecycle hooks for collection: ${collectionUID}`);
+
+      strapi.db.lifecycles.subscribe({
+        models: [collectionUID],
+
+        afterCreate: async (event) => {
+          await strapi
+            .service('plugin::elasticsearch.virtual-collections-indexer')
+            .handleTriggerEvent(event);
+        },
+
+        afterUpdate: async (event) => {
+          await strapi
+            .service('plugin::elasticsearch.virtual-collections-indexer')
+            .handleTriggerEvent(event);
+        },
+
+        afterDelete: async (event) => {
+          await strapi
+            .service('plugin::elasticsearch.virtual-collections-indexer')
+            .handleTriggerEvent(event);
+        },
+      });
+    });
+
+    // Initialize indexes
+    await strapi.service('plugin::elasticsearch.virtual-collections-indexer').initializeIndexes();
+  };
+
+  return plugin;
+};