jekyll-algolia 1.0.0 → 1.6.0

data/lib/errors/unknown_settings.txt

@@ -0,0 +1,12 @@
+E:[✗ Error] Unknown setting
+E:
+E:The jekyll-algolia plugin could not correctly configure your index as some of the settings passed are not recognized.
+W:
+W:It seems that one of the custom index settings you defined was not recognized by the API and was rejected:
+W:  {setting_name}: {setting_value}
+I:
+I:Make sure the setting name and value are correct. You can find a list of all the available settings with their documentation in our API reference:
+I:  https://www.algolia.com/doc/api-reference/api-parameters/
+I:Or specifically for this setting:
+I:  https://www.algolia.com/doc/api-reference/api-parameters/{setting_name}/
+I:

data/lib/jekyll-algolia.rb

@@ -1,22 +1,25 @@
 # frozen_string_literal: true
 
 require 'jekyll/commands/algolia'
+require 'date'
 
 module Jekyll
   # Requirable file, loading all dependencies.
   # Methods here are called by the main `jekyll algolia` command
   module Algolia
-    require 'jekyll/algolia/version'
-    require 'jekyll/algolia/utils'
-    require 'jekyll/algolia/hooks'
     require 'jekyll/algolia/configurator'
-    require 'jekyll/algolia/logger'
     require 'jekyll/algolia/error_handler'
-    require 'jekyll/algolia/file_browser'
     require 'jekyll/algolia/extractor'
+    require 'jekyll/algolia/file_browser'
+    require 'jekyll/algolia/hooks'
     require 'jekyll/algolia/indexer'
+    require 'jekyll/algolia/logger'
+    require 'jekyll/algolia/progress_bar'
+    require 'jekyll/algolia/shrinker'
+    require 'jekyll/algolia/utils'
+    require 'jekyll/algolia/version'
 
-    @config = {}
+    MissingCredentialsError = Class.new(StandardError)
 
     # Public: Init the Algolia module
     #
@@ -26,14 +29,46 @@ module Jekyll
     # The gist of the plugin works by instanciating a Jekyll site,
     # monkey-patching its `write` method and building it.
     def self.init(config = {})
-      @config = config
-      @site = Jekyll::Algolia::Site.new(@config)
+      # Monkey patch Jekyll and external plugins
+      load_overwrites
+
+      config = Configurator.init(config).config
+      @site = Jekyll::Algolia::Site.new(config)
+
+      unless Configurator.assert_valid_credentials
+        raise(
+          MissingCredentialsError,
+          "One or more credentials were not found for site at: #{@site.source}"
+        )
+      end
+
+      Configurator.warn_of_deprecated_options
 
-      exit 1 unless Configurator.assert_valid_credentials
+      if Configurator.dry_run?
+        Logger.log('W:==== THIS IS A DRY RUN ====')
+        Logger.log('W:  - No records will be pushed to your index')
+        Logger.log('W:  - No settings will be updated on your index')
+      end
 
       self
     end
 
+    # Public: Monkey patch Jekyll and external plugins so they don't interfere
+    # with our plugin
+    #
+    # Note: This is only loaded when running `jekyll algolia` so should not have
+    # any impact on regular builds
+    def self.load_overwrites
+      require 'jekyll/algolia/overwrites/githubpages-configuration'
+      require 'jekyll/algolia/overwrites/jekyll-algolia-site'
+      require 'jekyll/algolia/overwrites/jekyll-document'
+      require 'jekyll/algolia/overwrites/jekyll-paginate-pager'
+      require 'jekyll/algolia/overwrites/jekyll-tags-link'
+
+      # Register our own tags to overwrite the default tags
+      Liquid::Template.register_tag('link', JekyllAlgoliaLink)
+    end
+
     # Public: Run the main Algolia module
     #
     # Actually "process" the site, which will acts just like a regular `jekyll
@@ -43,65 +78,15 @@ module Jekyll
     # Note: The internal list of files to be processed will only be created when
     # calling .process
     def self.run
+      Logger.log('I:Processing site...')
       @site.process
     end
 
-    # Public: Get access to the Jekyll config
-    #
-    # All other classes will need access to this config, so we make it publicly
-    # accessible
-    def self.config
-      @config
-    end
-
     # Public: Get access to the Jekyll site
     #
     # Tests will need access to the inner Jekyll website so we expose it here
     def self.site
       @site
     end
-
-    # A Jekyll::Site subclass that overrides #write from the parent class to
-    # create JSON records out of rendered documents and push those records
-    # to Algolia instead of writing files to disk.
-    class Site < Jekyll::Site
-      def write
-        if Configurator.dry_run?
-          Logger.log('W:==== THIS IS A DRY RUN ====')
-          Logger.log('W:  - No records will be pushed to your index')
-          Logger.log('W:  - No settings will be updated on your index')
-        end
-
-        records = []
-        files = []
-        each_site_file do |file|
-          # Skip files that should not be indexed
-          is_indexable = FileBrowser.indexable?(file)
-          unless is_indexable
-            Logger.verbose("W:Skipping #{file.path}")
-            next
-          end
-
-          path = FileBrowser.path_from_root(file)
-          Logger.verbose("I:Extracting records from #{path}")
-          file_records = Extractor.run(file)
-
-          files << file
-          records += file_records
-        end
-
-        # Applying the user hook on the whole list of records
-        records = Hooks.apply_all(records)
-
-        # Adding a unique objectID to each record
-        records.map! do |record|
-          Extractor.add_unique_object_id(record)
-        end
-
-        Logger.verbose("I:Found #{files.length} files")
-
-        Indexer.run(records)
-      end
-    end
   end
 end

data/lib/jekyll/algolia/configurator.rb

@@ -6,64 +6,93 @@ module Jekyll
     module Configurator
       include Jekyll::Algolia
 
+      @config = {}
+
       # Algolia default values
       ALGOLIA_DEFAULTS = {
         'extensions_to_index' => nil,
         'files_to_exclude' => nil,
         'nodes_to_index' => 'p',
         'indexing_batch_size' => 1000,
-        'indexing_mode' => 'diff',
+        'max_record_size' => 10_000,
         'settings' => {
-          'distinct' => true,
-          'attributeForDistinct' => 'url',
-          'attributesForFaceting' => %w[
-            searchable(tags)
-            searchable(type)
-            searchable(title)
+          # Searchable attributes
+          'searchableAttributes' => %w[
+            title
+            headings
+            unordered(content)
+            collection,categories,tags
           ],
+          # Custom Ranking
           'customRanking' => [
             'desc(date)',
-            'desc(weight.heading)',
-            'asc(weight.position)'
+            'desc(custom_ranking.heading)',
+            'asc(custom_ranking.position)'
           ],
-          'highlightPreTag' => '<em class="ais-Highlight">',
-          'highlightPostTag' => '</em>',
-          'searchableAttributes' => %w[
-            title
-            hierarchy.lvl0
-            hierarchy.lvl1
-            hierarchy.lvl2
-            hierarchy.lvl3
-            hierarchy.lvl4
-            hierarchy.lvl5
-            unordered(content)
-            collection,unordered(categories),unordered(tags)
+          'unretrievableAttributes' => [
+            'custom_ranking'
           ],
-          # We want to allow highlight in more keys than what we search on
+          # Highlight
           'attributesToHighlight' => %w[
             title
-            hierarchy.lvl0
-            hierarchy.lvl1
-            hierarchy.lvl2
-            hierarchy.lvl3
-            hierarchy.lvl4
-            hierarchy.lvl5
+            headings
             content
             html
             collection
             categories
             tags
+          ],
+          'highlightPreTag' => '<em class="ais-Highlight">',
+          'highlightPostTag' => '</em>',
+          # Snippet
+          'attributesToSnippet' => %w[
+            content:55
+          ],
+          'snippetEllipsisText' => '…',
+          # Distinct
+          'distinct' => true,
+          'attributeForDistinct' => 'url',
+          # Faceting
+          'attributesForFaceting' => %w[
+            type
+            searchable(collection)
+            searchable(categories)
+            searchable(tags)
+            searchable(title)
           ]
         }
       }.freeze
 
+      # Public: Init the configurator with the Jekyll config
+      #
+      # config - The config passed by the `jekyll algolia` command. Default to
+      # the default Jekyll config
+      def self.init(config = nil)
+        # Use the default Jekyll configuration if none specified. Silence the
+        # warning about no config set
+        Logger.silent { config = Jekyll.configuration } if config.nil?
+
+        @config = config
+
+        @config = disable_other_plugins(@config)
+
+        self
+      end
+
+      # Public: Access to the global configuration object
+      #
+      # This is a method around @config so we can mock it in the tests
+      def self.config
+        @config
+      end
+
       # Public: Get the value of a specific Jekyll configuration option
       #
       # key - Key to read
       #
       # Returns the value of this configuration option, nil otherwise
       def self.get(key)
-        Jekyll::Algolia.config[key]
+        config[key]
       end
 
       # Public: Get the value of a specific Algolia configuration option, or
@@ -74,7 +103,7 @@ module Jekyll
       # Returns the value of this option, or the default value
       def self.algolia(key)
         config = get('algolia') || {}
-        value = config[key] || ALGOLIA_DEFAULTS[key]
+        value = config[key].nil? ? ALGOLIA_DEFAULTS[key] : config[key]
 
         # No value found but we have a method to define the default value
         if value.nil? && respond_to?("default_#{key}")
@@ -120,27 +149,22 @@ module Jekyll
         ENV['ALGOLIA_INDEX_NAME'] || algolia('index_name')
       end
 
+      # Public: Return the name of the index used to store the object ids
+      def self.index_object_ids_name
+        "#{index_name}_object_ids"
+      end
+
       # Public: Get the index settings
       #
       # This will be a merge of default settings and the one defined in the
       # _config.yml file
       def self.settings
+        return {} if algolia('settings') == false
+
         user_settings = algolia('settings') || {}
         ALGOLIA_DEFAULTS['settings'].merge(user_settings)
       end
 
-      # Public: Return the current indexing mode
-      #
-      # Default mode is `diff`, but users can configure their own by updating
-      # the `indexing_mode` config in _config.yml. The only other authorized
-      # value is `atomic`. If an unrecognized mode is defined, it defaults to
-      # `diff`.
-      def self.indexing_mode
-        mode = algolia('indexing_mode') || ALGOLIA_DEFAULTS['indexing_mode']
-        return 'diff' unless %w[diff atomic].include?(mode)
-        mode
-      end
-
       # Public: Check that all credentials are set
       #
       # Returns true if everything is ok, false otherwise. Will display helpful
@@ -162,7 +186,8 @@ module Jekyll
       # Markdown files can have many different extensions. We keep the one
       # defined in the Jekyll config
       def self.default_extensions_to_index
-        ['html'] + get('markdown_ext').split(',')
+        markdown_ext = get('markdown_ext') || ''
+        ['html'] + markdown_ext.split(',')
       end
 
       # Public: Setting a default value to ignore index.html/index.md files in
@@ -175,7 +200,7 @@ module Jekyll
       # User can still add it by manually specifying a `files_to_exclude` to an
       # empty array
       def self.default_files_to_exclude
-        algolia('extensions_to_index').map do |extension|
+        extensions_to_index.map do |extension|
           "index.#{extension}"
         end
       end
@@ -186,6 +211,7 @@ module Jekyll
       def self.verbose?
         value = get('verbose')
         return true if value == true
+
         false
       end
 
@@ -195,8 +221,75 @@ module Jekyll
       def self.dry_run?
         value = get('dry_run')
         return true if value == true
+
+        false
+      end
+
+      # Public: Returns true if the command should always update the settings
+      #
+      # When set to true, the index settings will always be updated, no matter
+      # if they've been modified or not
+      def self.force_settings?
+        value = get('force_settings')
+        return true if value == true
+
         false
       end
+
+      # Public: Returns a list of extensions to index
+      #
+      # Will use default values or read the algolia.extensions_to_index key.
+      # Accepts both an array or a comma-separated list
+      def self.extensions_to_index
+        extensions = algolia('extensions_to_index')
+        return [] if extensions.nil?
+
+        extensions = extensions.split(',') if extensions.is_a? String
+        extensions
+      end
+
+      # Public: Disable features from other Jekyll plugins that might interfere
+      # with the indexing
+      # Note that if other jekyll plugins are defined as part of the
+      # :jekyll_plugins group in the Gemfile, we might be able to override them
+      # using .load_overwrites in jekyll-algolia.rb.
+      # If they are simply required in Gemfile, then we might need to revert
+      # their values to nil values from here
+      def self.disable_other_plugins(config)
+        # Disable archive pages from jekyll-archives
+        config.delete('jekyll-archives')
+
+        # Disable pagination from jekyll-paginate
+        config.delete('paginate')
+
+        # Disable pagination for jekyll-paginate-v2
+        config['pagination'] = {} unless config['pagination'].is_a?(Hash)
+        config['pagination']['enabled'] = false
+
+        # Disable autopages for jekyll-paginate-v2
+        config['autopages'] = {} unless config['autopages'].is_a?(Hash)
+        config['autopages']['enabled'] = false
+
+        # Disable tags from jekyll-tagging
+        config.delete('tag_page_dir')
+        config.delete('tag_page_layout')
+
+        config
+      end
+
+      # Public: Check for any deprecated config option and warn the user
+      def self.warn_of_deprecated_options
+        # indexing_mode is no longer used
+        return if algolia('indexing_mode').nil?
+
+        # rubocop:disable Metrics/LineLength
+        Logger.log('I:')
+        Logger.log('W:[jekyll-algolia] You are using the algolia.indexing_mode option which has been deprecated in v1.1')
+        Logger.log('I:    Indexing is now always using an atomic diff algorithm.')
+        Logger.log('I:    This option is no longer necessary, you can remove it from your _config.yml')
+        Logger.log('I:')
+        # rubocop:enable Metrics/LineLength
+      end
     end
   end
 end

data/lib/jekyll/algolia/error_handler.rb

@@ -18,7 +18,6 @@ module Jekyll
       # happened to the display
       def self.stop(error, context = {})
         Logger.verbose("E:[jekyll-algolia] Raw error: #{error}")
-        Logger.verbose("E:[jekyll-algolia] Context: #{context}")
 
         identified_error = identify(error, context)
 
@@ -47,10 +46,10 @@ module Jekyll
       def self.identify(error, context = {})
         known_errors = %w[
           unknown_application_id
-          invalid_credentials_for_tmp_index
           invalid_credentials
-          record_too_big
-          unknown_settings
+          record_too_big_api
+          too_many_records
+          unknown_setting
           invalid_index_name
         ]
 
@@ -58,6 +57,7 @@ module Jekyll
         known_errors.each do |potential_error|
           error_check = send("#{potential_error}?", error, context)
           next if error_check == false
+
           return {
             name: potential_error,
             details: error_check
@@ -156,29 +156,6 @@ module Jekyll
         { 'application_id' => app_id }
       end
 
-      # Public: Check if credentials specifically can't access the _tmp index
-      #
-      # _context - Not used
-      #
-      # If the error happens on a _tmp folder, it might mean that the key does
-      # not have access to the _tmp indices and the error message will reflect
-      # that.
-      def self.invalid_credentials_for_tmp_index?(error, _context = {})
-        details = error_hash(error.message)
-
-        index_name_tmp = details['index_name']
-        if details['message'] != 'Index not allowed with this API key' ||
-           index_name_tmp !~ /_tmp$/
-          return false
-        end
-
-        {
-          'application_id' => Configurator.application_id,
-          'index_name' => Configurator.index_name,
-          'index_name_tmp' => index_name_tmp
-        }
-      end
-
       # Public: Check if the credentials are working
       #
       # _context - Not used
@@ -186,45 +163,41 @@ module Jekyll
       # Application ID and API key submitted don't match any credentials known
       def self.invalid_credentials?(error, _context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         if details['message'] != 'Invalid Application-ID or API key'
           return false
         end
 
         {
-          'application_id' => details['application_id']
+          'application_id' => details['application_id'],
+          'index_name' => Configurator.index_name,
+          'index_object_ids_name' => Configurator.index_object_ids_name
         }
       end
 
       # Public: Check if the sent records are not too big
       #
-      # context[:records] - list of records to push
+      # context[:records] - list of records sent in the batch
       #
-      # Records cannot weight more that 10Kb. If we're getting this error it
-      # means that one of the records is too big, so we'll try to give
-      # informations about it so the user can debug it.
-      def self.record_too_big?(error, context = {})
+      # One of the sent record is too big and has been rejected by the API. This
+      # should not happen as we proactively check for record size before pushing
+      # them. If it still happens it means that the value set in max_record_size
+      # is not matching the value in the plan.
+      def self.record_too_big_api?(error, _context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         message = details['message']
         return false if message !~ /^Record .* is too big .*/
 
-        # Getting the record size
-        size, = /.*size=(.*) bytes.*/.match(message).captures
-        size = Filesize.from("#{size} B").pretty
-        object_id = details['objectID']
-
-        # Getting record details
-        record = Utils.find_by_key(context[:records], :objectID, object_id)
+        record_size, = /.*size=(.*) bytes.*/.match(message).captures
+        record_size_readable = Filesize.from("#{record_size}B").to_s('Kb')
+        max_record_size = Configurator.algolia('max_record_size')
 
         {
-          'object_id' => object_id,
-          'object_title' => record[:title],
-          'object_url' => record[:url],
-          'object_hint' => record[:content][0..100],
-          'nodes_to_index' => Configurator.algolia('nodes_to_index'),
-          'size' => size,
-          'size_limit' => '10 Kb'
+          'record_size' => record_size_readable,
+          'max_record_size' => max_record_size
         }
       end
 
@@ -235,8 +208,9 @@ module Jekyll
       # The API will block any call that tries to update a setting value that is
       # not available. We'll tell the user which one so they can fix their
       # issue.
-      def self.unknown_settings?(error, context = {})
+      def self.unknown_setting?(error, context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         message = details['message']
         return false if message !~ /^Invalid object attributes.*/
@@ -257,6 +231,7 @@ module Jekyll
       # Some characters are forbidden in index names
       def self.invalid_index_name?(error, _context = {})
         details = error_hash(error.message)
+        return false if details == false
 
         message = details['message']
         return false if message !~ /^indexName is not valid.*/
@@ -265,6 +240,19 @@ module Jekyll
           'index_name' => Configurator.index_name
         }
       end
+
+      # Public: Check if the application has too many records
+      #
+      # We're trying to push too many records and it goes over quota
+      def self.too_many_records?(error, _context = {})
+        details = error_hash(error.message)
+        return false if details == false
+
+        message = details['message']
+        return false if message !~ /^Record quota exceeded.*/
+
+        {}
+      end
     end
   end
 end