@redpanda-data/docs-extensions-and-macros 3.1.0 → 3.1.2

package/extensions/algolia-indexer/generate-index.js

@@ -188,7 +188,8 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
       objectID: urlPath + page.pub.url,
       titles: titles,
       keywords: keywords,
-      _tags: [tag]
+      type: 'Doc',
+      _tags: [tag, 'docs']
     }
 
     algolia[cname][version].push(indexItem)

package/extensions/algolia-indexer/index.js

@@ -3,9 +3,12 @@
 const generateIndex = require('./generate-index')
 const chalk = require('chalk')
 const algoliasearch = require('algoliasearch')
+const http = require('http')
+const https = require('https')
 const fs = require('fs')
 const path = require('path')
 const _ = require('lodash')
+process.env.UV_THREADPOOL_SIZE=16
 
 /**
  * Algolia indexing for an Antora documentation site.
@@ -26,8 +29,14 @@ function register({
   var client
   var index
 
-  // Connect and authenticate with Algolia
-  client = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY)
+  const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 100 });
+  const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 100 });
+
+  // Connect and authenticate with Algolia using the custom agent
+  client = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY, {
+    httpAgent: httpAgent,
+    httpsAgent: httpsAgent
+  })
   index = client.initIndex(process.env.ALGOLIA_INDEX_NAME)
 
   if (Object.keys(unknownOptions).length) {
@@ -58,6 +67,7 @@ function register({
 
     let totalObjectsToUpdate = 0
     let totalObjectsToAdd = 0
+    const objectsToDelete = []
 
     for (const c of Object.keys(algolia)) {
       for (const v of Object.keys(algolia[c])) {
@@ -72,66 +82,60 @@ function register({
               objectsToUpdate.push(obj)
               totalObjectsToUpdate++
             }
+            existingObjectsMap.delete(obj.objectID)
           } else {
             objectsToAdd.push(obj)
             totalObjectsToAdd++
           }
         }
 
-        // Upload new records only if the objects have been updated or they are new.
-        // See https://www.algolia.com/doc/api-reference/api-methods/save-objects/
-        if (objectsToUpdate.length) {
-          index.saveObjects(objectsToUpdate)
-            .then(() => {
-              logger.info(`Updated records for ${c} version ${v}`)
-            })
-            .catch(error => {
-              logger.error(`Error updating objects to Algolia: ${error.message}`)
-            })
-        }
-        if (objectsToAdd.length) {
-          index.saveObjects(objectsToAdd)
-            .then(() => {
-              logger.info(`Added records for ${c} version ${v}`)
-            })
-            .catch(error => {
-              logger.error(`Error adding objects to Algolia: ${error.message}`)
-            })
-        }
+        const addObjectActions = objectsToAdd.map(object => ({
+          action: 'addObject',
+          indexName: process.env.ALGOLIA_INDEX_NAME,
+          body: object
+        }));
+
+        const updateObjectActions = objectsToUpdate.map(object => ({
+          action: 'updateObject',
+          indexName: process.env.ALGOLIA_INDEX_NAME,
+          body: object
+        }));
 
-        siteCatalog.addFile({
-          mediaType: 'application/json',
-          contents: Buffer.from(JSON.stringify(algolia[c][v])),
-          src: { stem: `algolia-${c}` },
-          out: { path: `algolia-${c}-${v}.json` },
-          pub: { url: `/algolia-${c}-${v}.json`, rootPath: '' }
-        })
+        const batchActions = [...addObjectActions, ...updateObjectActions];
+
+        // Upload new records only if the objects have been updated or they are new.
+        // See https://www.algolia.com/doc/api-reference/api-methods/batch/?client=javascript
+        await client.multipleBatch(batchActions).then(() => {
+          console.log('Batch operations completed successfully');
+        }).catch(error => {
+          logger.error(`Error uploading records to Algolia: ${error.message}`);
+        });
       }
     }
 
-    index.setSettings({
-      attributesForFaceting: ['version', 'product']
-    })
+    for (const [objectID, obj] of existingObjectsMap) {
+      if (obj.type === 'Doc' && !obj._tags.includes('apis')) {
+        objectsToDelete.push(objectID)
+      }
+    }
+    if (objectsToDelete.length > 0) {
+      console.log(objectsToDelete)
+      await index.deleteObjects(objectsToDelete).then(() => {
+        console.log(`Deleted ${objectsToDelete.length} outdated records`);
+      }).catch(error => {
+        logger.error(`Error deleting records from Algolia: ${error.message}`);
+      });
+    }
 
     console.log(chalk.green('Updated records:' + totalObjectsToUpdate))
     console.log(chalk.green('New records:' + totalObjectsToAdd))
 
-    totalObjectsToAdd === 0 && totalObjectsToUpdate === 0 && console.log(chalk.green('No changes made'))
-
-    try {
-      let recordCount = 0
-
-      await index.browseObjects({
-        query: '',
-        batch: batch => {
-          recordCount += batch.length
-        }
-      })
+    totalObjectsToAdd === 0 && totalObjectsToUpdate === 0 && console.log(chalk.green('No new records uploaded or existing records updated'))
+  })
 
-      console.log(chalk.green('Total records:', recordCount))
-    } catch (err) {
-      logger.error(JSON.stringify(err))
-    }
+  process.on('exit', () => {
+    httpAgent.destroy()
+    httpsAgent.destroy()
   })
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "3.1.0",
+  "version": "3.1.2",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",