jekyll-algolia 1.1.5 → 1.2.0

data/lib/jekyll/algolia/configurator.rb

@@ -15,45 +15,49 @@ module Jekyll
         'nodes_to_index' => 'p',
         'indexing_batch_size' => 1000,
         'settings' => {
-          'distinct' => true,
-          'attributeForDistinct' => 'url',
-          'attributesForFaceting' => %w[
-            searchable(tags)
-            searchable(type)
-            searchable(title)
+          # Searchable attributes
+          'searchableAttributes' => %w[
+            title
+            headings
+            unordered(content)
+            collection,categories,tags
           ],
+          # Custom Ranking
           'customRanking' => [
             'desc(date)',
-            'desc(weight.heading)',
-            'asc(weight.position)'
+            'desc(custom_ranking.heading)',
+            'asc(custom_ranking.position)'
           ],
-          'highlightPreTag' => '<em class="ais-Highlight">',
-          'highlightPostTag' => '</em>',
-          'searchableAttributes' => %w[
-            title
-            hierarchy.lvl0
-            hierarchy.lvl1
-            hierarchy.lvl2
-            hierarchy.lvl3
-            hierarchy.lvl4
-            hierarchy.lvl5
-            unordered(content)
-            collection,unordered(categories),unordered(tags)
+          'unretrievableAttributes' => [
+            'custom_ranking'
           ],
-          # We want to allow highlight in more keys than what we search on
+          # Highlight
           'attributesToHighlight' => %w[
             title
-            hierarchy.lvl0
-            hierarchy.lvl1
-            hierarchy.lvl2
-            hierarchy.lvl3
-            hierarchy.lvl4
-            hierarchy.lvl5
+            headings
             content
             html
             collection
             categories
             tags
+          ],
+          'highlightPreTag' => '<em class="ais-Highlight">',
+          'highlightPostTag' => '</em>',
+          # Snippet
+          'attributesToSnippet' => %w[
+            content:55
+          ],
+          'snippetEllipsisText' => '…',
+          # Distinct
+          'distinct' => true,
+          'attributeForDistinct' => 'url',
+          # Faceting
+          'attributesForFaceting' => %w[
+            type
+            searchable(collection)
+            searchable(categories)
+            searchable(tags)
+            searchable(title)
           ]
         }
       }.freeze
@@ -68,7 +72,6 @@ module Jekyll
         Logger.silent { config = Jekyll.configuration } if config.nil?
 
         @config = config
-        @config['exclude'] = files_excluded_from_render
 
         @config = disable_other_plugins(@config)
 
@@ -212,23 +215,14 @@ module Jekyll
         false
       end
 
-      # Public: List of files to exclude from the Jekyll build
+      # Public: Returns true if the command should always update the settings
       #
-      # We skip all files usually ignored by Jekyll, plus any file that should
-      # not be indexed.
-      def self.files_excluded_from_render
-        site_exclude = get('exclude') || []
-        algolia_exclude = algolia('files_to_exclude') || []
-
-        excluded_files = site_exclude + algolia_exclude
-
-        # 404 pages are not Jekyll defaults but a convention adopted by GitHub
-        # pages. We don't want to index those.
-        # https://help.github.com/articles/creating-a-custom-404-page-for-your-github-pages-site/
-        excluded_files << '404.html'
-        excluded_files << '404.md'
-
-        excluded_files
+      # When set to true, the index settings will always be updated, no matter
+      # if they've been modified or not
+      def self.force_settings?
+        value = get('force_settings')
+        return true if value == true
+        false
       end
 
       # Public: Disable features from other Jekyll plugins that might interfere

data/lib/jekyll/algolia/extractor.rb

@@ -12,7 +12,7 @@ module Jekyll
       #
       # file - The Jekyll file to process
       def self.run(file)
-        # Getting all hierarchical nodes from the HTML input
+        # Getting all nodes from the HTML input
         raw_records = extract_raw_records(file.content)
         # Getting file metadata
         shared_metadata = FileBrowser.metadata(file)
@@ -47,7 +47,7 @@ module Jekyll
       end
 
       # Public: Extract raw records from the file, including content for each
-      # node to index and hierarchy
+      # node and its headings
       #
       # content - The HTML content to parse
       def self.extract_raw_records(content)

data/lib/jekyll/algolia/file_browser.rb

@@ -14,46 +14,72 @@ module Jekyll
     module FileBrowser
       include Jekyll::Algolia
 
-      # Public: Check if the specified file is a static Jekyll asset
-      #
-      # file - The Jekyll file
-      #
-      # We don't index static assets (js, css, images)
-      def self.static_file?(file)
-        file.is_a?(Jekyll::StaticFile)
-      end
-
       # Public: Return the absolute path of a Jekyll file
       #
       # file - The Jekyll file to inspect
-      #
-      # Jekyll handles the .path property of some files as relative to the root
-      # (pages) or as an absolute paths (posts and static assets). We make sure
-      # we have a consistent way of accessing it
-      def self.absolute_path(file)
-        pathname = Pathname.new(file.path)
+      def self.absolute_path(filepath)
+        pathname = Pathname.new(filepath)
         return pathname.cleanpath.to_s if pathname.absolute?
 
-        File.expand_path(File.join(Configurator.get('source'), file.path))
+        File.expand_path(File.join(Configurator.get('source'), filepath))
       end
 
       # Public: Return the path of a Jekyll file relative to the Jekyll source
       #
       # file - The Jekyll file to inspect
-      #
-      # Jekyll handles the .path property of some files as relative to the root
-      # (pages) or as an absolute paths (posts and static assets). We make sure
-      # we have a consistent way of accessing it
-      def self.relative_path(file)
-        pathname = Pathname.new(file.path)
-        return file.path if pathname.relative?
+      def self.relative_path(filepath)
+        pathname = Pathname.new(filepath)
+        config_source = Configurator.get('source') || ''
+        jekyll_source = Pathname.new(File.expand_path(config_source))
+
+        # Removing any starting ./
+        if pathname.relative?
+          fullpath = File.expand_path(File.join(jekyll_source, pathname))
+          return fullpath.gsub(%r{^#{jekyll_source}/}, '')
+        end
 
-        jekyll_source = Pathname.new(
-          File.expand_path(Configurator.get('source'))
-        )
         pathname.relative_path_from(jekyll_source).cleanpath.to_s
       end
 
+      # Public: Check if the file should be indexed
+      #
+      # file - The Jekyll file
+      #
+      # There are many reasons a file should not be indexed. We need to exclude
+      # all the static assets, only keep the actual content.
+      def self.indexable?(file)
+        return false if static_file?(file)
+        return false if is_404?(file)
+        return false unless allowed_extension?(file)
+        return false if excluded_from_config?(file)
+        return false if excluded_from_hook?(file)
+
+        true
+      end
+
+      # Public: Check if the specified file is a static Jekyll asset
+      #
+      # file - The Jekyll file
+      #
+      # We don't index static assets (js, css, images)
+      def self.static_file?(file)
+        file.is_a?(Jekyll::StaticFile)
+      end
+
+      # Public: Check if the file is a 404 error page
+      #
+      # file - The Jekyll file
+      #
+      # 404 pages are not Jekyll defaults but a convention adopted by GitHub
+      # pages. We don't want to index those.
+      # Source: https://help.github.com/articles/creating-a-custom-404-page-for-your-github-pages-site/
+      #
+      # rubocop:disable Naming/PredicateName
+      def self.is_404?(file)
+        ['404.md', '404.html'].include?(File.basename(file.path))
+      end
+      # rubocop:enable Naming/PredicateName
+
       # Public: Check if the file has one of the allowed extensions
       #
       # file - The Jekyll file
@@ -74,18 +100,13 @@ module Jekyll
       def self.excluded_from_config?(file)
         excluded_patterns = Configurator.algolia('files_to_exclude')
         jekyll_source = Configurator.get('source')
+        path = absolute_path(file.path)
 
-        # Transform the glob patterns into a real list of files
-        excluded_files = []
-        Dir.chdir(jekyll_source) do
-          excluded_patterns.each do |pattern|
-            Dir.glob(pattern).each do |match|
-              excluded_files << File.expand_path(match)
-            end
-          end
+        excluded_patterns.each do |pattern|
+          pattern = File.expand_path(File.join(jekyll_source, pattern))
+          return true if File.fnmatch(pattern, path, File::FNM_PATHNAME)
         end
-
-        excluded_files.include?(absolute_path(file))
+        false
       end
 
       # Public: Check if the file has been excluded by running a custom user
@@ -96,20 +117,6 @@ module Jekyll
         Hooks.should_be_excluded?(file.path)
       end
 
-      # Public: Check if the file should be indexed
-      #
-      # file - The Jekyll file
-      #
-      # There are many reasons a file should not be indexed. We need to exclude
-      # all the static assets, only keep the actual content.
-      def self.indexable?(file)
-        return false if static_file?(file)
-        return false unless allowed_extension?(file)
-        return false if excluded_from_hook?(file)
-
-        true
-      end
-
       # Public: Return a hash of all the file metadata
       #
       # file - The Jekyll file
@@ -121,6 +128,8 @@ module Jekyll
         raw_data = raw_data(file)
         specific_data = {
           collection: collection(file),
+          tags: tags(file),
+          categories: categories(file),
           date: date(file),
           excerpt_html: excerpt_html(file),
           excerpt_text: excerpt_text(file),
@@ -153,6 +162,11 @@ module Jekyll
         end
         data.delete('excerpt')
 
+        # Delete other keys added by Jekyll that are not in the front-matter and
+        # not needed for search
+        data.delete('draft')
+        data.delete('ext')
+
         # Convert all values to a version that can be serialized to JSON
         data = Utils.jsonify(data)
 
@@ -184,30 +198,39 @@ module Jekyll
         file.url
       end
 
-      # Public: Returns a timestamp of the file date
+      # Public: Returns the list of tags of a file, defaults to an empty array
       #
       # file - The Jekyll file
+      def self.tags(file)
+        file.data['tags'] || []
+      end
+
+      # Public: Returns the list of tags of a file, defaults to an empty array
       #
-      # All collections (including posts) will have a date taken either from the
-      # front-matter or the filename prefix. If none is set, Jekyll will use the
-      # current date.
+      # file - The Jekyll file
+      def self.categories(file)
+        file.data['categories'] || []
+      end
+
+      # Public: Returns a timestamp of the file date
       #
-      # For pages, only dates defined in the front-matter will be used.
+      # file - The Jekyll file
       #
-      # Note that because the default date is the current one if none is
-      # defined, we have to make sure the date is actually nil when we index it.
-      # Otherwise the diff indexing mode will think that records have changed
-      # while they haven't.
+      # Posts have their date coming from the filepath, or the front-matter.
+      # Pages and other collection items can only have a date set in
+      # front-matter.
       def self.date(file)
-        date = file.data['date']
-        return nil if date.nil?
+        # Collections get their date from .date, while pages read it from .data.
+        # Jekyll by default will set the date of collection to the current date,
+        # but we overwrote this.
+        date = if file.respond_to?(:date)
+                 file.date
+               else
+                 file.data['date']
+               end
 
-        # The date is *exactly* the time where the `jekyll algolia` was run.
-        # What a coincidence! It's a safe bet to assume that the original date
-        # was nil and has been overwritten by Jekyll
-        return nil if date.to_i == Jekyll::Algolia.start_time.to_i
-
-        date.to_i
+        return nil if date.nil?
+        date.to_time.to_i
       end
 
       # Public: Returns the raw excerpt of a file, directly as returned by
@@ -224,7 +247,7 @@ module Jekyll
           return file.data['excerpt'].to_s
         end
       rescue StandardError
-        return nil
+        nil
       end
 
       # Public: Returns the HTML version of the excerpt

data/lib/jekyll/algolia/indexer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'algoliasearch'
+require 'yaml'
+require 'algolia_html_extractor'
 
 module Jekyll
   module Algolia
@@ -17,8 +19,16 @@ module Jekyll
           application_id: Configurator.application_id,
           api_key: Configurator.api_key
         )
+        @index = ::Algolia::Index.new(Configurator.index_name)
 
         set_user_agent
+
+        self
+      end
+
+      # Public: Returns the Algolia index object
+      def self.index
+        @index
       end
 
       # Public: Set the User-Agent to send to the API
@@ -38,33 +48,11 @@ module Jekyll
         ::Algolia.set_extra_header('User-Agent', user_agent)
       end
 
-      # Public: Returns an Algolia Index object from an index name
-      #
-      # index_name - String name of the index
-      def self.index(index_name)
-        ::Algolia::Index.new(index_name)
-      end
-
-      # Public: Check if an index exists
-      #
-      # index_name - Name of the index
-      #
-      # Note: there is no API endpoint to do that, so we try to get the settings
-      # instead, which will fail if the index does not exist
-      def self.index?(index_name)
-        index(index_name).get_settings
-        return true
-      rescue StandardError
-        return false
-      end
-
       # Public: Returns an array of all the objectIDs in the index
       #
-      # index - Algolia Index to target
-      #
       # The returned array is sorted. It won't have any impact on the way it is
       # processed, but makes debugging easier when comparing arrays is needed.
-      def self.remote_object_ids(index)
+      def self.remote_object_ids
         list = []
         Logger.verbose(
           "I:Inspecting existing records in index #{index.name}..."
@@ -88,49 +76,22 @@ module Jekyll
         records.map { |record| record[:objectID] }.compact.sort
       end
 
-      # Public: Update settings of the index
-      #
-      # index - The Algolia Index
-      #
-      # Does nothing in dry run mode
-      # Settings will only be updated in the first push, and if custom settings
-      # are defined in _config.yml. Otherwise, they are left untouched, allowing
-      # users to configure them through their dashboard.
-      def self.update_settings(index)
-        has_custom_settings = !Configurator.algolia('settings').nil?
-        index_exists = index?(index.name)
-
-        # No need to update the settings if the index is already configured and
-        # the user did not specify custom settings
-        return if index_exists && !has_custom_settings
-
-        Logger.verbose('I:Updating settings')
-        return if Configurator.dry_run?
-        settings = Configurator.settings
-        begin
-          index.set_settings!(settings)
-        rescue StandardError => error
-          ErrorHandler.stop(error, settings: settings)
-        end
-      end
-
       # Public: Update records of the index
       #
-      # index_name - The Algolia index
       # old_records_ids - Ids of records to delete from the index
       # new_records - Records to add to the index
       #
       # Note: All operations will be done in one batch, assuring an atomic
       # update
       # Does nothing in dry run mode
-      def self.update_records(index_name, old_records_ids, new_records)
+      def self.update_records(old_records_ids, new_records)
         # Stop if nothing to change
         if old_records_ids.empty? && new_records.empty?
-          Logger.log('I:Nothing to index. Your content is already up to date.')
+          Logger.log('I:Content is already up to date.')
           return
         end
 
-        Logger.log("I:Updating records in index #{index_name}...")
+        Logger.log("I:Updating records in index #{index.name}...")
         Logger.log("I:Records to delete: #{old_records_ids.length}")
         Logger.log("I:Records to add:    #{new_records.length}")
         return if Configurator.dry_run?
@@ -141,19 +102,31 @@ module Jekyll
         operations = []
         old_records_ids.each do |object_id|
           operations << {
-            action: 'deleteObject', indexName: index_name,
+            action: 'deleteObject', indexName: index.name,
             body: { objectID: object_id }
           }
         end
         operations += new_records.map do |new_record|
-          { action: 'addObject', indexName: index_name, body: new_record }
+          { action: 'addObject', indexName: index.name, body: new_record }
         end
 
         # Run the batches in slices if they are too large
         batch_size = Configurator.algolia('indexing_batch_size')
-        operations.each_slice(batch_size) do |slice|
+        slices = operations.each_slice(batch_size).to_a
+
+        should_have_progress_bar = (slices.length > 1)
+        if should_have_progress_bar
+          progress_bar = ProgressBar.create(
+            total: slices.length,
+            format: 'Pushing records (%j%%) |%B|'
+          )
+        end
+
+        slices.each do |slice|
           begin
             ::Algolia.batch!(slice)
+
+            progress_bar.increment if should_have_progress_bar
           rescue StandardError => error
             records = slice.map do |record|
               record[:body]
@@ -163,6 +136,106 @@ module Jekyll
         end
       end
 
+      # Public: Get a unique settingID for the current settings
+      #
+      # The settingID is generated as a hash of the current settings. As it will
+      # be stored in the userData key of the resulting config, we exclude that
+      # key from the hashing.
+      def self.local_setting_id
+        settings = Configurator.settings
+        settings.delete('userData')
+        AlgoliaHTMLExtractor.uuid(settings)
+      end
+
+      # Public: Get the settings of the remote index
+      #
+      # In case the index is not accessible, it will return nil
+      def self.remote_settings
+        index.get_settings
+      rescue StandardError
+        nil
+      end
+
+      # Public: Smart update of the settings of the index
+      #
+      # This will first compare the settings about to be pushed with the
+      # settings already pushed. It will compare userData.settingID for that.
+      # If the settingID is the same, we don't push as this won't change
+      # anything. We will still check if the remote config seem to have been
+      # manually altered though, and warn the user that this is not the
+      # preferred way of doing so.
+      #
+      # If the settingID are not matching, it means our config is different, so
+      # we push it, overriding the settingID for next push.
+      def self.update_settings
+        current_remote_settings = remote_settings || {}
+        remote_setting_id = current_remote_settings.dig('userData', 'settingID')
+
+        settings = Configurator.settings
+        setting_id = local_setting_id
+
+        are_settings_forced = Configurator.force_settings?
+
+        # The config we're about to push is the same we pushed previously. We
+        # won't push again.
+        if setting_id == remote_setting_id && !are_settings_forced
+          Logger.log('I:Settings are already up to date.')
+          # Check if remote config has been changed outside of the plugin, so we
+          # can warn users that they should not alter their config from outside
+          # of the plugin.
+          current_remote_settings.delete('userData')
+          changed_keys = Utils.diff_keys(settings, current_remote_settings)
+          unless changed_keys.nil?
+            warn_of_manual_dashboard_editing(changed_keys)
+          end
+
+          return
+        end
+
+        # Settings have changed, we push them
+        settings['userData'] = {
+          'settingID' => setting_id,
+          'pluginVersion' => VERSION
+        }
+
+        Logger.log("I:Updating settings of index #{index.name}")
+        return if Configurator.dry_run?
+        set_settings(settings)
+      end
+
+      # Public: Set new settings to an index
+      #
+      # Will dispatch to the error handler if it fails
+      # rubocop:disable Naming/AccessorMethodName
+      def self.set_settings(settings)
+        index.set_settings!(settings)
+      rescue StandardError => error
+        ErrorHandler.stop(error, settings: settings)
+      end
+      # rubocop:enable Naming/AccessorMethodName
+
+      # Public: Warn users that they have some settings manually configured in
+      # their dashboard
+      #
+      # When users change some settings in their dashboard, those settings might
+      # get overwritten by the pluging. We can't prevent that, but we can warn
+      # them when we detect they changed something.
+      def self.warn_of_manual_dashboard_editing(changed_keys)
+        # Transform the hash into readable YAML
+        yaml_lines = changed_keys
+                     .to_yaml(indentation: 2)
+                     .split("\n")[1..-1]
+        yaml_lines.map! do |line|
+          line = line.gsub(/^ */) { |spaces| ' ' * spaces.length }
+          line = line.gsub('- ', '  - ')
+          "W:    #{line}"
+        end
+        Logger.known_message(
+          'settings_manually_edited',
+          settings: yaml_lines.join("\n")
+        )
+      end
+
       # Public: Push all records to Algolia and configure the index
       #
       # records - Records to push
@@ -180,14 +253,11 @@ module Jekyll
           exit 1
         end
 
-        index_name = Configurator.index_name
-        index = index(index_name)
-
         # Update settings
-        update_settings(index)
+        update_settings
 
         # Getting list of objectID in remote and locally
-        remote_ids = remote_object_ids(index)
+        remote_ids = remote_object_ids
         local_ids = local_object_ids(records)
 
         # Getting list of what to add and what to delete
@@ -196,7 +266,7 @@ module Jekyll
         new_records = records.select do |record|
           new_records_ids.include?(record[:objectID])
         end
-        update_records(index_name, old_records_ids, new_records)
+        update_records(old_records_ids, new_records)
 
         Logger.log('I:✔ Indexing complete')
       end