jekyll-algolia 1.0.0 → 1.6.0

data/lib/jekyll/algolia/logger.rb

@@ -4,24 +4,44 @@ module Jekyll
   module Algolia
     # Display helpful error messages
     module Logger
+      # Public: Silence all Jekyll log output in this block
+      # Usage:
+      #   Logger.silence do
+      #     # whatever Jekyll code here
+      #   end
+      #
+      # This is especially useful when Jekyll is too talkative about what is
+      # loggued. It works by redefining Jekyll.logger.write to a noop
+      # temporarily and re-attributing the original method once finished.
+      def self.silent
+        initial_method = Jekyll.logger.method(:write)
+        Utils.monkey_patch(Jekyll.logger, :write, proc { |*args| })
+        begin
+          yield
+        ensure
+          Utils.monkey_patch(Jekyll.logger, :write, initial_method)
+        end
+      end
+
       # Public: Displays a log line
       #
       # line - Line to display. Expected to be of the following format:
       #   "X:Your content"
       # Where X is either I, W or E for marking respectively an info, warning or
       # error display
-      def self.log(line)
-        type, content = /^(I|W|E):(.*)/.match(line).captures
+      def self.log(input)
+        type, content = /^(I|W|E):(.*)/m.match(input).captures
         logger_mapping = {
           'E' => :error,
           'I' => :info,
           'W' => :warn
         }
 
-        # Jekyll logger tries to center log lines, so we force a consistent
-        # width of 80 chars
-        content = content.ljust(80, ' ')
-        Jekyll.logger.send(logger_mapping[type], content)
+        # Display by chunk of 80-characters lines
+        lines = Utils.split_lines(content, 80)
+        lines.each do |line|
+          Jekyll.logger.send(logger_mapping[type], line)
+        end
       end
 
       # Public: Only display a log line if verbose mode is enabled
@@ -29,9 +49,20 @@ module Jekyll
       # line - The line to display, following the same format as .log
       def self.verbose(line)
         return unless Configurator.verbose?
+
         log(line)
       end
 
+      # Public: Write the specified content to a file in the source directory
+      #
+      # filename - the file basename
+      # content - the actual content of the file
+      def self.write_to_file(filename, content)
+        filepath = File.join(Configurator.get('source'), filename)
+        File.write(filepath, content)
+        filepath
+      end
+
       # Public: Displays a helpful error message for one of the knows errors
       #
       # message_id: A string identifying a know message
@@ -42,14 +73,14 @@ module Jekyll
       def self.known_message(message_id, metadata = {})
         file = File.expand_path(
           File.join(
-            __dir__, '../../..', 'errors', "#{message_id}.txt"
+            __dir__, '../..', 'errors', "#{message_id}.txt"
           )
         )
 
         # Convert all variables
         content = File.open(file).read
         metadata.each do |key, value|
-          content = content.gsub("{#{key}}", value)
+          content = content.gsub("{#{key}}", value.to_s)
         end
 
         # Display each line differently

data/lib/jekyll/algolia/overwrites/githubpages-configuration.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module GitHubPages
+  # The github-pages gem will automatically disable every plugin that is not in
+  # the whitelist of plugins allowed by GitHub. This includes any plugin defined
+  # in the `_plugins` folder as well.
+  #
+  # Users of the jekyll-algolia plugin will use custom plugins in _plugins to
+  # define custom hooks to modify the indexing. If they happen to have the
+  # github-pages gem installed at the same time, those hooks will never be
+  # executed.
+  #
+  # The GitHub Pages gem prevent access to custom plugins by doing two things:
+  # - forcing safe mode
+  # - loading custom plugins from a random dir
+  #
+  # We cancel those by disabling safe mode and forcing back plugins to be read
+  # from ./_plugins.
+  #
+  # This file will only be loaded when running `jekyll algolia`, so it won't
+  # interfere with the regular usage of `jekyll build`
+  class Configuration
+    class << self
+      def set!(site)
+        config = effective_config(site.config)
+        config['safe'] = false
+        config['plugins_dir'] = '_plugins'
+        site.config = config
+      end
+    end
+  end
+end

data/lib/jekyll/algolia/overwrites/jekyll-algolia-site.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module Algolia
+    # A Jekyll::Site subclass that overrides process from the parent class to
+    # create JSON records out of rendered documents and push those records to
+    # Algolia instead of writing files to disk.
+    class Site < Jekyll::Site
+      # We expose a way to reset the collection, as it will be needed in the
+      # tests
+      attr_writer :collections
+
+      attr_reader :original_site_files
+
+      # Public: Overwriting the parent method
+      #
+      # This will prepare the website, gathering all files, excluding the one we
+      # don't need to index, then render them (converting to HTML), the finally
+      # calling `push` to push to Algolia
+      def process
+        # Default Jekyll preflight
+        reset
+        read
+        generate
+
+        # Removing all files that won't be indexed, so we don't waste time
+        # rendering them
+        keep_only_indexable_files
+
+        # Starting the rendering progress bar
+        init_rendering_progress_bar
+
+        # Converting them to HTML
+        render
+
+        # Pushing them Algolia
+        push
+      end
+
+      # Public: Return the number of pages/documents to index
+      def indexable_item_count
+        count = @pages.length
+        @collections.each_value { |collection| count += collection.docs.length }
+        count
+      end
+
+      # Public: Init the rendering progress bar, incrementing it for each
+      # rendered item
+      #
+      # This uses Jekyll post_render hooks, listening to both pages and
+      # documents
+      def init_rendering_progress_bar
+        progress_bar = ProgressBar.create(
+          total: indexable_item_count,
+          format: 'Rendering to HTML (%j%%) |%B|'
+        )
+        Jekyll::Hooks.register [:pages, :documents], :post_render do
+          progress_bar.increment
+        end
+      end
+
+      # Public: Filtering a list of items to only keep the one that are
+      # indexable.
+      #
+      # items - List of Pages/Documents
+      #
+      # Note: It also sets the layout to nil, to further speed up the rendering
+      def indexable_list(items)
+        new_list = []
+        items.each do |item|
+          next unless FileBrowser.indexable?(item)
+
+          item.data = {} if item.data.nil?
+          item.data['layout'] = nil
+          new_list << item
+        end
+        new_list
+      end
+
+      # Public: Removing non-indexable Pages, Posts and Documents from the
+      # internals
+      def keep_only_indexable_files
+        @original_site_files = {
+          pages: @pages,
+          collections: @collections,
+          static_files: @static_files
+        }
+
+        @pages = indexable_list(@pages)
+
+        # Applying to each collections
+        @collections.each_value do |collection|
+          collection.docs = indexable_list(collection.docs)
+        end
+
+        # Remove all static files
+        @static_files = []
+      end
+
+      # Public: Extract records from every file and index them
+      def push
+        records = []
+        files = []
+        progress_bar = ProgressBar.create(
+          total: indexable_item_count,
+          format: 'Extracting records (%j%%) |%B|'
+        )
+        each_site_file do |file|
+          # Even if we cleared the list of documents/pages beforehand, some
+          # files might still sneak up to this point (like static files added to
+          # a collection directory), so we check again if they can really be
+          # indexed.
+          next unless FileBrowser.indexable?(file)
+
+          path = FileBrowser.relative_path(file.path)
+
+          Logger.verbose("I:Extracting records from #{path}")
+          file_records = Extractor.run(file)
+
+          files << file
+          records += file_records
+
+          progress_bar.increment
+        end
+
+        # Applying the user hook on the whole list of records
+        records = Hooks.apply_all(records, self)
+
+        # Shrinking records to force them to fit under the max record size
+        # limit, or displaying an error message if not possible
+        max_record_size = Configurator.algolia('max_record_size')
+        # We take into account the objectID that will be added in the form of:
+        # "objectID": "16cd998991cc40d92402b0b4e6c55e8a"
+        object_id_attribute_length = 46
+        max_record_size -= object_id_attribute_length
+        records.map! do |record|
+          Shrinker.fit_to_size(record, max_record_size)
+        end
+
+        # Adding a unique objectID to each record
+        records.map! do |record|
+          Extractor.add_unique_object_id(record)
+        end
+
+        Logger.verbose("I:Found #{files.length} files")
+
+        Indexer.run(records)
+      end
+    end
+  end
+end

data/lib/jekyll/algolia/overwrites/jekyll-document.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Jekyll
+  # Overwriting the Jekyll::Document class
+  class Document
+    # By default, Jekyll will set the current date (time of build) to any
+    # collection item. This will break our diff algorithm, so we monkey patch
+    # this call to return nil if no date is defined instead.
+    def date
+      data['date'] || nil
+    end
+  end
+end

data/lib/jekyll/algolia/overwrites/jekyll-paginate-pager.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module Paginate
+    # Disable pagination from jekyll-paginate
+    #
+    # This plugin will create pages that contain a list of all items to
+    # paginate. Those pages won't contain any interesting data to be indexed
+    # (as it will be duplicated content of the real pages), but will still
+    # take time to generate.
+    #
+    # By monkey-patching the plugin, we force it to be disabled
+    # https://github.com/jekyll/jekyll-paginate/blob/master/lib/jekyll-paginate/pager.rb#L22
+    class Pager
+      def self.pagination_enabled?(_site)
+        false
+      end
+    end
+  end
+end

data/lib/jekyll/algolia/overwrites/jekyll-tags-link.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# The default `link` tag allow to link to a specific page, using its relative
+# path. Because we might not be indexing the destination of the link, we might
+# not have the representation of the page in our data. If that happens, the
+# `link` tag fails.
+#
+# To fix that we'll overwrite the default `link` tag to loop over a backup copy
+# of the original files (before we clean it for indexing)
+#
+# https://github.com/algolia/jekyll-algolia/issues/62
+class JekyllAlgoliaLink < Jekyll::Tags::Link
+  def render(context)
+    original_files = context.registers[:site].original_site_files
+
+    original_files[:pages].each do |page|
+      return page.url if page.relative_path == @relative_path
+    end
+
+    original_files[:collections].each_value do |collection|
+      collection.docs.each do |item|
+        return item.url if item.relative_path == @relative_path
+      end
+    end
+
+    original_files[:static_files].each do |asset|
+      return asset.url if asset.relative_path == @relative_path
+      return asset.url if asset.relative_path == "/#{@relative_path}"
+    end
+
+    '/'
+  end
+end

data/lib/jekyll/algolia/progress_bar.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'progressbar'
+require 'ostruct'
+
+module Jekyll
+  module Algolia
+    # Module to push records to Algolia and configure the index
+    module ProgressBar
+      include Jekyll::Algolia
+
+      def self.should_be_silenced?
+        Configurator.verbose?
+      end
+
+      def self.create(options)
+        if should_be_silenced?
+          fake_bar = OpenStruct.new
+          fake_bar.increment = nil
+          return fake_bar
+        end
+
+        ::ProgressBar.create(options)
+      end
+    end
+  end
+end

data/lib/jekyll/algolia/shrinker.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'json'
+module Jekyll
+  module Algolia
+    # Module to shrink a record so it fits in the plan quotas
+    module Shrinker
+      include Jekyll::Algolia
+
+      # Public: Get the byte size of the object once converted to JSON
+      # - record: The record to estimate
+      def self.size(record)
+        record.to_json.bytesize
+      end
+
+      # Public: Attempt to reduce the size of the record by reducing the size of
+      # the less needed attributes
+      #
+      # - raw_record: The record to attempt to reduce
+      # - max_size: The max size to achieve in bytes
+      #
+      # The excerpts are the attributes most subject to being reduced. We'll go
+      # as far as removing them if there is no other choice.
+      def self.fit_to_size(raw_record, max_size)
+        return raw_record if size(raw_record) <= max_size
+
+        # No excerpt, we can't shrink it
+        if !raw_record.key?(:excerpt_html) || !raw_record.key?(:excerpt_text)
+          return stop_with_error(raw_record)
+        end
+
+        record = raw_record.clone
+
+        # We replace the HTML excerpt with the textual one
+        record[:excerpt_html] = record[:excerpt_text]
+        return record if size(record) <= max_size
+
+        # We half the excerpts
+        excerpt_words = record[:excerpt_text].split(/\s+/)
+        shortened_excerpt = excerpt_words[0...excerpt_words.size / 2].join(' ')
+        record[:excerpt_text] = shortened_excerpt
+        record[:excerpt_html] = shortened_excerpt
+        return record if size(record) <= max_size
+
+        # We remove the excerpts completely
+        record.delete(:excerpt_text)
+        record.delete(:excerpt_html)
+        return record if size(record) <= max_size
+
+        # Still too big, we fail
+        stop_with_error(record)
+      end
+
+      # Public: Stop the current indexing process and display details about the
+      # record that is too big to be pushed
+      #
+      # - record: The record causing the error
+      #
+      # This will display an error message and log the wrong record in a file in
+      # the source directory
+      def self.stop_with_error(record)
+        record_size = size(record)
+        record_size_readable = Filesize.from("#{record_size}B").to_s('Kb')
+        max_record_size = Configurator.algolia('max_record_size')
+        max_record_size_readable = Filesize
+                                   .from("#{max_record_size}B").to_s('Kb')
+
+        probable_wrong_keys = readable_largest_record_keys(record)
+
+        # Writing the full record to disk for inspection
+        record_log_path = Logger.write_to_file(
+          'jekyll-algolia-record-too-big.log',
+          JSON.pretty_generate(record)
+        )
+
+        details = {
+          'object_title' => record[:title],
+          'object_url' => record[:url],
+          'probable_wrong_keys' => probable_wrong_keys,
+          'record_log_path' => record_log_path,
+          'nodes_to_index' => Configurator.algolia('nodes_to_index'),
+          'record_size' => record_size_readable,
+          'max_record_size' => max_record_size_readable
+        }
+
+        Logger.known_message('record_too_big', details)
+
+        stop_process
+      end
+
+      # Public: Returns a string explaining which attributes are the largest in
+      # the record
+      #
+      # record - The record hash to analyze
+      def self.readable_largest_record_keys(record)
+        keys = Hash[record.map { |key, value| [key, value.to_s.length] }]
+        largest_keys = keys.sort_by { |_, value| value }.reverse[0..2]
+        output = []
+        largest_keys.each do |key, size|
+          size = Filesize.from("#{size} B").to_s('Kb')
+          output << "#{key} (#{size})"
+        end
+        output.join(', ')
+      end
+
+      # Public: Stop the current process
+      def self.stop_process
+        exit 1
+      end
+    end
+  end
+end