jekyll-algolia 1.2.7 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6243fd7aa41bd8a50531f03d81c2162ff2c0c8a1
-  data.tar.gz: 16da8529592e1cf701cce7781588486d41a11390
+  metadata.gz: cf200adeeada4a57c74a46db83ca98f29fe69bde
+  data.tar.gz: 4098806de6e5cc74d79021b99bc8875fad5cacee
 SHA512:
-  metadata.gz: b95eb1944bca76b4fd0520f3f9ad6c29b0763c515dc0160edfb2a8e5086b9da1b9827741e258f5b27334efef4e49e4c86235dd02c2f8a4ad69f95b30dc254a2e
-  data.tar.gz: e40caac42d9d8793bc2d15025c0a3f017b9ee04235d84bfd4bb4fb2b6e392159edb7b290333752e373ec96931c7f3ef7ef7b46e2ea4ddeb1b3e826b16db262cb
+  metadata.gz: 25b04ae1214a048234d7ad9512a00939cc99fcc4d35e025a08bdfd826c4267d355f21d67a2e559fde018d25a5fd45cb611c27cc4d6c846f0641ead7c034a7107
+  data.tar.gz: dd36674e3de277bf3fa62daf3b24fb1e165d68979f8ad6b2935235587779d660e4ae0ffb29453168fde47a75c29890a08cb5061a4c2bc54606d04a72fd581f4d

data/lib/errors/invalid_credentials.txt

@@ -3,6 +3,9 @@ E:
 E:The jekyll-algolia plugin could not connect to your application ID using the API key your provided.
 W:
 W:Make sure your API key has access to your {application_id} application.
+W:It should also have the rights to push to the following indices:
+W:   - {index_name}
+W:   - {index_object_ids_name}
 I:
 I:You can find your API key in your Algolia dashboard here:
 I:   https://www.algolia.com/licensing

data/lib/errors/too_many_records.txt

@@ -0,0 +1,14 @@
+E:[✗ Error] Too many records
+E:
+E:The jekyll-algolia plugin could not push your records because it exceeds the maximum number of records allowed in your current plan.
+W:
+W:Community plans can host up to 10k records and Essential plans starts at 50k.
+W:
+W:Check our pricing page for more details:
+W:   https://www.algolia.com/pricing
+W:
+I:You might want to upgrade your plan or exclude records from indexing using the `files_to_exclude` option:
+I:   https://community.algolia.com/jekyll-algolia/options.html#files-to-exclude
+I:
+I:If you're having trouble solving this issue, feel free to file a bug on GitHub, ideally with a link to a repository where we can reproduce the issue as well as the APPID you're trying to push to.
+I:  https://github.com/algolia/jekyll-algolia/issues

data/lib/jekyll/algolia/configurator.rb

@@ -148,6 +148,11 @@ module Jekyll
         ENV['ALGOLIA_INDEX_NAME'] || algolia('index_name')
       end
 
+      # Public: Return the name of the index used to store the object ids
+      def self.index_object_ids_name
+        "#{index_name}_object_ids"
+      end
+
       # Public: Get the index settings
       #
       # This will be a merge of default settings and the one defined in the

data/lib/jekyll/algolia/error_handler.rb

@@ -48,7 +48,8 @@ module Jekyll
           unknown_application_id
           invalid_credentials
           record_too_big
-          unknown_settings
+          too_many_records
+          unknown_setting
           invalid_index_name
         ]
 
@@ -134,6 +135,24 @@ module Jekyll
         hash
       end
 
+      # Public: Returns a string explaining which attributes are the largest in
+      # the record
+      #
+      # record - The record hash to analyze
+      #
+      # This will be used on the `record_too_big` error, to guide users in
+      # finding which record is causing trouble
+      def self.readable_largest_record_keys(record)
+        keys = Hash[record.map { |key, value| [key, value.to_s.length] }]
+        largest_keys = keys.sort_by { |_, value| value }.reverse[0..2]
+        output = []
+        largest_keys.each do |key, size|
+          size = Filesize.from("#{size} B").to_s('Kb')
+          output << "#{key} (#{size})"
+        end
+        output.join(', ')
+      end
+
       # Public: Check if the application id is available
       #
       # _context - Not used
@@ -161,34 +180,19 @@ module Jekyll
       # Application ID and API key submitted don't match any credentials known
       def self.invalid_credentials?(error, _context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         if details['message'] != 'Invalid Application-ID or API key'
           return false
         end
 
         {
-          'application_id' => details['application_id']
+          'application_id' => details['application_id'],
+          'index_name' => Configurator.index_name,
+          'index_object_ids_name' => Configurator.index_object_ids_name
         }
       end
 
-      # Public: Returns a string explaining which attributes are the largest in
-      # the record
-      #
-      # record - The record hash to analyze
-      #
-      # This will be used on the `record_too_big` error, to guide users in
-      # finding which record is causing trouble
-      def self.readable_largest_record_keys(record)
-        keys = Hash[record.map { |key, value| [key, value.to_s.length] }]
-        largest_keys = keys.sort_by { |_, value| value }.reverse[0..2]
-        output = []
-        largest_keys.each do |key, size|
-          size = Filesize.from("#{size} B").to_s('Kb')
-          output << "#{key} (#{size})"
-        end
-        output.join(', ')
-      end
-
       # Public: Check if the sent records are not too big
       #
       # context[:records] - list of records sent in the batch
@@ -198,6 +202,7 @@ module Jekyll
       # informations about it so the user can debug it.
       def self.record_too_big?(error, context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         message = details['message']
         return false if message !~ /^Record .* is too big .*/
@@ -207,8 +212,11 @@ module Jekyll
         size = Filesize.from("#{size} B").to_s('Kb')
         object_id = details['objectID']
 
-        # Getting record details
-        record = Utils.find_by_key(context[:records], :objectID, object_id)
+        # Finding the record in all the operations
+        operation = context[:operations].find do |o|
+          o[:action] == 'addObject' && o[:body][:objectID] == object_id
+        end
+        record = operation[:body]
         probable_wrong_keys = readable_largest_record_keys(record)
 
         # Writing the full record to disk for inspection
@@ -236,8 +244,9 @@ module Jekyll
       # The API will block any call that tries to update a setting value that is
       # not available. We'll tell the user which one so they can fix their
       # issue.
-      def self.unknown_settings?(error, context = {})
+      def self.unknown_setting?(error, context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         message = details['message']
         return false if message !~ /^Invalid object attributes.*/
@@ -258,6 +267,7 @@ module Jekyll
       # Some characters are forbidden in index names
       def self.invalid_index_name?(error, _context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         message = details['message']
         return false if message !~ /^indexName is not valid.*/
@@ -266,6 +276,19 @@ module Jekyll
           'index_name' => Configurator.index_name
         }
       end
+
+      # Public: Check if the application has too many records
+      #
+      # We're trying to push too many records and it goes over quota
+      def self.too_many_records?(error, _context = {})
+        details = error_hash(error.message)
+        return false if details == false
+
+        message = details['message']
+        return false if message !~ /^Record quota exceeded.*/
+
+        {}
+      end
     end
   end
 end

data/lib/jekyll/algolia/indexer.rb

@@ -4,6 +4,7 @@ require 'algoliasearch'
 require 'yaml'
 require 'algolia_html_extractor'
 
+# rubocop:disable Metrics/ModuleLength
 module Jekyll
   module Algolia
     # Module to push records to Algolia and configure the index
@@ -11,15 +12,15 @@ module Jekyll
       include Jekyll::Algolia
 
       # Public: Init the module
-      #
-      # This call will instanciate the Algolia API client, set the custom
-      # User Agent and give an easy access to the main index
       def self.init
         ::Algolia.init(
           application_id: Configurator.application_id,
           api_key: Configurator.api_key
         )
-        @index = ::Algolia::Index.new(Configurator.index_name)
+        index_name = Configurator.index_name
+        @index = ::Algolia::Index.new(index_name)
+        index_object_ids_name = Configurator.index_object_ids_name
+        @index_object_ids = ::Algolia::Index.new(index_object_ids_name)
 
         set_user_agent
 
@@ -31,6 +32,42 @@ module Jekyll
         @index
       end
 
+      # Public: Returns the Algolia index used to store object ids
+      def self.index_object_ids
+        @index_object_ids
+      end
+
+      # Public: Check if an index exists
+      #
+      # index - Index to check
+      #
+      # Note: there is no API endpoint to do that, so we try to get the settings
+      # instead, which will fail if the index does not exist
+      def self.index_exist?(index)
+        index.get_settings
+        true
+      rescue StandardError
+        false
+      end
+
+      # Public: Get the number of records in an index
+      #
+      # index - Index to check
+      #
+      # Note: We'll do an empty query search, to match everything, but we'll
+      # only return the objectID and one element, to get the shortest response
+      # possible. It will still contain the nbHits
+      def self.record_count(index)
+        index.search(
+          '',
+          attributesToRetrieve: 'objectID',
+          distinct: false,
+          hitsPerPage: 1
+        )['nbHits']
+      rescue StandardError
+        0
+      end
+
       # Public: Set the User-Agent to send to the API
       #
       # Every integrations should follow the "YYY Integration" pattern, and
@@ -48,27 +85,71 @@ module Jekyll
         ::Algolia.set_extra_header('User-Agent', user_agent)
       end
 
-      # Public: Returns an array of all the objectIDs in the index
+      # Public: Get an array of all object IDs stored in the main index
       #
-      # The returned array is sorted. It won't have any impact on the way it is
-      # processed, but makes debugging easier when comparing arrays is needed.
-      def self.remote_object_ids
+      # Note: As this will be slow (grabbing them 1000 at a time), we display
+      # a progress bar.
+      def self.remote_object_ids_from_main_index
+        Logger.verbose("I:Inspecting existing records in index #{index.name}")
+
         list = []
-        Logger.verbose(
-          "I:Inspecting existing records in index #{index.name}..."
+
+        # As it might take some time, we display a progress bar
+        progress_bar = ProgressBar.create(
+          total: record_count(index),
+          format: 'Inspecting existing records (%j%%) |%B|'
         )
         begin
-          index.browse(attributesToRetrieve: 'objectID') do |hit|
+          index.browse(
+            attributesToRetrieve: 'objectID',
+            hitsPerPage: 1000
+          ) do |hit|
             list << hit['objectID']
+            progress_bar.increment
           end
         rescue StandardError
-          # The index might not exist if it's the first time we use the plugin
-          # so we'll consider that it means there are no records there
           return []
         end
+
         list.sort
       end
 
+      # Public: Get an array of all the object ids, stored in the dedicated
+      # index
+      #
+      # Note: This will be very fast. Each record contain 100 object id, so it
+      # will fit in one call each time.
+      def self.remote_object_ids_from_dedicated_index
+        list = []
+        begin
+          index_object_ids.browse(
+            attributesToRetrieve: 'content',
+            hitsPerPage: 1000
+          ) do |hit|
+            list += hit['content']
+          end
+        rescue StandardError
+          return []
+        end
+
+        list.sort
+      end
+
+      # Public: Returns an array of all the objectIDs in the index
+      #
+      # Note: We use a dedicated index to store the objectIDs for faster
+      # browsing, but if the index does not exist we read the main index.
+      def self.remote_object_ids
+        Logger.log('I:Getting list of existing records')
+
+        # Fast version, using the dedicated index
+        has_dedicated_index = index_exist?(index_object_ids)
+        return remote_object_ids_from_dedicated_index if has_dedicated_index
+
+        # Slow version, browsing the full index
+        remote_object_ids_from_main_index
+      end
+
       # Public: Returns an array of the local objectIDs
       #
       # records - Array of all local records
@@ -78,38 +159,84 @@ module Jekyll
 
       # Public: Update records of the index
       #
-      # old_records_ids - Ids of records to delete from the index
-      # new_records - Records to add to the index
+      # records - All records extracted from Jekyll
       #
       # Note: All operations will be done in one batch, assuring an atomic
       # update
       # Does nothing in dry run mode
-      def self.update_records(old_records_ids, new_records)
+      def self.update_records(records)
+        # Getting list of objectID in remote and locally
+        remote_ids = remote_object_ids
+        local_ids = local_object_ids(records)
+
+        # Making a diff, to see what to add and what to delete
+        ids_to_delete = remote_ids - local_ids
+        ids_to_add = local_ids - remote_ids
+
+        # What changes should we do to the indexes?
+        has_records_to_update = !ids_to_delete.empty? || !ids_to_add.empty?
+        has_dedicated_index = index_exist?(index_object_ids)
+
         # Stop if nothing to change
-        if old_records_ids.empty? && new_records.empty?
+        if !has_records_to_update && has_dedicated_index
           Logger.log('I:Content is already up to date.')
           return
         end
 
-        Logger.log("I:Updating records in index #{index.name}...")
-        Logger.log("I:Records to delete: #{old_records_ids.length}")
-        Logger.log("I:Records to add:    #{new_records.length}")
-        return if Configurator.dry_run?
-
-        # We group delete and add operations into the same batch. Delete
-        # operations should still come first, to avoid hitting an overquota too
-        # soon
+        # We group all operations into one batch
         operations = []
-        old_records_ids.each do |object_id|
-          operations << {
-            action: 'deleteObject', indexName: index.name,
-            body: { objectID: object_id }
-          }
+
+        # We update records only if there are records to update
+        if has_records_to_update
+          Logger.log("I:Updating records in index #{index.name}...")
+          Logger.log("I:Records to delete: #{ids_to_delete.length}")
+          Logger.log("I:Records to add:    #{ids_to_add.length}")
+
+          # Transforming ids into real records to add
+          records_by_id = Hash[records.map { |r| [r[:objectID], r] }]
+          records_to_add = ids_to_add.map { |id| records_by_id[id] }
+
+          # Deletion operations come first, to avoid hitting an overquota too
+          # soon if it can be avoided
+          ids_to_delete.each do |object_id|
+            operations << {
+              action: 'deleteObject', indexName: index.name,
+              body: { objectID: object_id }
+            }
+          end
+          # Then we add the new records
+          operations += records_to_add.map do |new_record|
+            { action: 'addObject', indexName: index.name, body: new_record }
+          end
         end
-        operations += new_records.map do |new_record|
-          { action: 'addObject', indexName: index.name, body: new_record }
+
+        # We update the dedicated index everytime we update records, but we also
+        # create it if it does not exist
+        should_update_dedicated_index = has_records_to_update ||
+                                        !has_dedicated_index
+        if should_update_dedicated_index
+          operations << { action: 'clear', indexName: index_object_ids.name }
+          local_ids.each_slice(100).each do |ids|
+            operations << {
+              action: 'addObject', indexName: index_object_ids.name,
+              body: { content: ids }
+            }
+          end
         end
 
+        execute_operations(operations)
+      end
+
+      # Public: Execute a serie of operations in a batch
+      #
+      # operations - Operations to batch
+      #
+      # Note: Will split the batch in several calls if too big, and will display
+      # a progress bar if this happens
+      def self.execute_operations(operations)
+        return if Configurator.dry_run?
+        return if operations.empty?
+
         # Run the batches in slices if they are too large
         batch_size = Configurator.algolia('indexing_batch_size')
         slices = operations.each_slice(batch_size).to_a
@@ -118,7 +245,7 @@ module Jekyll
         if should_have_progress_bar
           progress_bar = ProgressBar.create(
             total: slices.length,
-            format: 'Pushing records (%j%%) |%B|'
+            format: 'Updating index (%j%%) |%B|'
           )
         end
 
@@ -128,10 +255,7 @@ module Jekyll
 
             progress_bar.increment if should_have_progress_bar
           rescue StandardError => error
-            records = slice.map do |record|
-              record[:body]
-            end
-            ErrorHandler.stop(error, records: records)
+            ErrorHandler.stop(error, operations: slice)
           end
         end
       end
@@ -253,23 +377,12 @@ module Jekyll
           exit 1
         end
 
-        # Update settings
         update_settings
-
-        # Getting list of objectID in remote and locally
-        remote_ids = remote_object_ids
-        local_ids = local_object_ids(records)
-
-        # Getting list of what to add and what to delete
-        old_records_ids = remote_ids - local_ids
-        new_records_ids = local_ids - remote_ids
-        new_records = records.select do |record|
-          new_records_ids.include?(record[:objectID])
-        end
-        update_records(old_records_ids, new_records)
+        update_records(records)
 
         Logger.log('I:✔ Indexing complete')
       end
     end
   end
 end
+# rubocop:enable Metrics/ModuleLength

data/lib/jekyll/algolia/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module Algolia
-    VERSION = '1.2.7'
+    VERSION = '1.3.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-algolia
 version: !ruby/object:Gem::Version
-  version: 1.2.7
+  version: 1.3.0
 platform: ruby
 authors:
 - Tim Carry
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-28 00:00:00.000000000 Z
+date: 2018-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: algolia_html_extractor
@@ -292,6 +292,7 @@ files:
 - lib/errors/no_records_found.txt
 - lib/errors/record_too_big.txt
 - lib/errors/settings_manually_edited.txt
+- lib/errors/too_many_records.txt
 - lib/errors/unknown_application_id.txt
 - lib/errors/unknown_settings.txt
 - lib/jekyll-algolia.rb