@redpanda-data/docs-extensions-and-macros 3.0.13 → 3.0.14

package/extensions/algolia-indexer/generate-index.js

@@ -59,7 +59,7 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
       }
     )
 
-    // skip pages marked as "noindex" for "robots"
+    // Skip pages marked as "noindex" for "robots"
     const noindex = root.querySelector('meta[name=robots][content=noindex]')
     if (noindex) {
       continue
@@ -104,7 +104,7 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
     // Start handling the article content
     const article = root.querySelector('article.doc')
     if (!article) {
-      logger.warn(`Page is not an article...skipping ${page.pub.url}`)
+      logger.info(`Page is not an article...skipping ${page.pub.url}`)
       continue
     }
 

package/extensions/algolia-indexer/index.js

@@ -1,17 +1,18 @@
 'use strict'
 
-const generateIndex       = require('./generate-index')
-const chalk               = require('chalk')
-const algoliasearch       = require('algoliasearch');
-const fs                  = require('fs');
-const path                = require('path');
+const generateIndex = require('./generate-index')
+const chalk = require('chalk')
+const algoliasearch = require('algoliasearch')
+const fs = require('fs')
+const path = require('path')
+const _ = require('lodash')
 
 /**
  * Algolia indexing for an Antora documentation site.
  *
  * @module antora-algolia-indexer
  */
-function register ({
+function register({
   config: {
     indexLatestOnly,
     excludes,
@@ -22,65 +23,116 @@ function register ({
 
   if (!process.env.ALGOLIA_ADMIN_API_KEY || !process.env.ALGOLIA_APP_ID || !process.env.ALGOLIA_INDEX_NAME) return
 
-  var client;
-  var index;
+  var client
+  var index
 
   // Connect and authenticate with Algolia
-  client = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY);
-
-  // Create a new index and add a record
-  index = client.initIndex(process.env.ALGOLIA_INDEX_NAME);
+  client = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY)
+  index = client.initIndex(process.env.ALGOLIA_INDEX_NAME)
 
   if (Object.keys(unknownOptions).length) {
     const keys = Object.keys(unknownOptions)
     throw new Error(`Unrecognized option${keys.length > 1 ? 's' : ''} specified: ${keys.join(', ')}`)
   }
 
-  this.on('beforePublish', ({ playbook, siteCatalog, contentCatalog }) => {
+  this.on('beforePublish', async ({ playbook, siteCatalog, contentCatalog }) => {
     const algolia = generateIndex(playbook, contentCatalog, { indexLatestOnly, excludes, logger })
-    // write the Algolia indexes
-    var algoliaCount = 0
-    Object.keys(algolia).forEach((c) => {
-      Object.keys(algolia[c]).forEach((v) => {
-        algoliaCount += algolia[c][v].length
-        // Save all records to the index
-        index.saveObjects(algolia[c][v]).then(() => {
-          console.log(`Indexed ${c} version ${v}`)
-        }).catch(error => {
-          console.error(`Error saving objects to Algolia: ${error}`);
-        });
+    let existingObjectsMap = new Map()
+
+    // Save objects in a local cache to query later.
+    // Avoids sending multiple requests.
+    // browseObjects does not affect analytics or usage limits.
+    // See https://www.algolia.com/doc/api-reference/api-methods/browse/#about-this-method
+    try {
+      await index.browseObjects({
+        query: '',
+        batch: batch => {
+          for (const obj of batch) {
+            existingObjectsMap.set(obj.objectID, obj)
+          }
+        }
+      })
+    } catch (err) {
+      logger.error(JSON.stringify(err))
+    }
+
+    let totalObjectsToUpdate = 0
+    let totalObjectsToAdd = 0
+
+    for (const c of Object.keys(algolia)) {
+      for (const v of Object.keys(algolia[c])) {
+        const objectsToUpdate = []
+        const objectsToAdd = []
+
+        for (const obj of algolia[c][v]) {
+          const existingObject = existingObjectsMap.get(obj.objectID)
+
+          if (existingObject) {
+            if (!_.isEqual(existingObject, obj)) {
+              objectsToUpdate.push(obj)
+              totalObjectsToUpdate++
+            }
+          } else {
+            objectsToAdd.push(obj)
+            totalObjectsToAdd++
+          }
+        }
+
+        // Upload new records only if the objects have been updated or they are new.
+        // See https://www.algolia.com/doc/api-reference/api-methods/save-objects/
+        if (objectsToUpdate.length) {
+          index.saveObjects(objectsToUpdate)
+            .then(() => {
+              logger.info(`Updated records for ${c} version ${v}`)
+            })
+            .catch(error => {
+              logger.error(`Error updating objects to Algolia: ${error.message}`)
+            })
+        }
+        if (objectsToAdd.length) {
+          index.saveObjects(objectsToAdd)
+            .then(() => {
+              logger.info(`Added records for ${c} version ${v}`)
+            })
+            .catch(error => {
+              logger.error(`Error adding objects to Algolia: ${error.message}`)
+            })
+        }
+
         siteCatalog.addFile({
           mediaType: 'application/json',
-          contents: Buffer.from(
-            JSON.stringify(algolia[c][v])
-          ),
+          contents: Buffer.from(JSON.stringify(algolia[c][v])),
           src: { stem: `algolia-${c}` },
           out: { path: `algolia-${c}-${v}.json` },
-          pub: { url: `/algolia-${c}-${v}.json`, rootPath: '' },
+          pub: { url: `/algolia-${c}-${v}.json`, rootPath: '' }
         })
-      })
-    })
+      }
+    }
+
     index.setSettings({
-      attributesForFaceting: [
-        'version',
-        'product'
-      ]
+      attributesForFaceting: ['version', 'product']
     })
-    // Get and print the count of all records in the index
-    let recordCount = 0;
-    index
-      .browseObjects({
-        query: '', // for all records
+
+    console.log(chalk.green('Updated records:' + totalObjectsToUpdate))
+    console.log(chalk.green('New records:' + totalObjectsToAdd))
+
+    totalObjectsToAdd === 0 && totalObjectsToUpdate === 0 && console.log(chalk.green('No changes made'))
+
+    try {
+      let recordCount = 0
+
+      await index.browseObjects({
+        query: '',
         batch: batch => {
-          recordCount += batch.length;
+          recordCount += batch.length
         }
       })
-      .then(() => {
-        console.log('Total Records:', recordCount);
-      })
-      .catch(err => console.log(JSON.stringify(err)));
+
+      console.log(chalk.green('Total records:', recordCount))
+    } catch (err) {
+      logger.error(JSON.stringify(err))
+    }
   })
 }
 
 module.exports = { generateIndex, register }
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "3.0.13",
+  "version": "3.0.14",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",