jekyll-algolia 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 18ee9d2d725bb531703ddc05d9f31250959d81ff
-  data.tar.gz: 93115f75f4d41d301898b77809732453d5c195f2
+  metadata.gz: 145d078774c95a6e9dda69331854efa6fbc51672
+  data.tar.gz: 34e00c41dee01556fe49811604a7819a3e4b252a
 SHA512:
-  metadata.gz: fed961094843fdf503620f86fcb83313c96b68061ce0d170b1a66f018d60cf1edf0a68a9dd5df7f17dee5c2a6e6e4fa05e677ae5a743383c859c42f3443e6bba
-  data.tar.gz: d66ea9ca93b8cafc3749711f17d27f03753ddfc6f4de70d2cc94296e5209b9247f2e3bbbefbf924414e5fe3c380f13c13fac87f80b8eed9322ccb0338d2dad2b
+  metadata.gz: 3a6d8b20abbeba1af7ee8de0bf8a111d41d40f6119caa156a1db433b1ca502cdff793cc8ffde2691fa60389b29b270f6fe4e2a3afb2294414d88de3c3c699af2
+  data.tar.gz: 3bbcff1330a507c2f16a9c13f5f102741086bb34c362e71a746640f2ff55ec1f125985114bfe8598adb23f66c8ab71bb9de2e341168cb69c608242ae6e9e64d8

data/lib/errors/record_too_big.txt

@@ -5,20 +5,22 @@ E: the {size_limit} size limit.
 W:
 W: The plugin will create one record for each element matching your
 W: `nodes_to_index` value (currently set to "{nodes_to_index}"). Each record
-W: should not weight more than {size_limit}. One of your records weights {size} 
-W: and has been rejected.
+W: should not weight more than {size_limit}. 
+W: One of your records weights {size} and has been rejected.
 W:
-W: Here are more information about the rejected record:
-W: {
-W:   "objectID": "{object_id}",
-W:   "title": "{object_title}",
-W:   "url": "{object_url}",
-W:   "text": "{object_hint}…",
-W:   […]
-W: }
+W: objectID: {object_id}
+W: title:    {object_title}
+W: url:      {object_url}
+W:
+W: Most probable keys causing the issue:
+W:    {probable_wrong_keys}
+W:
+W: Complete log of the record has been extracted to:
+W:    {record_log_path}
 W: 
-I: This issue is sometimes caused by malformed HTML preventing the parser to
-I: correctly grab the content of the nodes.
+I: This issue can be caused by malformed HTML preventing the parser to correctly
+I: grab the content of the nodes. Double check that the page actually renders 
+I: correctly with a regular `jekyll build`.
 I:
 I: If you're having trouble solving this issue, feel free to file a bug on
 I: GitHub, ideally with a link to a repository where we can reproduce the issue.

data/lib/jekyll-algolia.rb

@@ -91,14 +91,15 @@ module Jekyll
         files = []
         Logger.log('I:Extracting records...')
         each_site_file do |file|
+          path = FileBrowser.relative_path(file)
+
           # Skip files that should not be indexed
           is_indexable = FileBrowser.indexable?(file)
           unless is_indexable
-            Logger.verbose("W:Skipping #{file.path}")
+            Logger.verbose("W:Skipping #{path}")
             next
           end
 
-          path = FileBrowser.path_from_root(file)
           Logger.verbose("I:Extracting records from #{path}")
           file_records = Extractor.run(file)
 

data/lib/jekyll/algolia/error_handler.rb

@@ -18,7 +18,6 @@ module Jekyll
       # happened to the display
       def self.stop(error, context = {})
         Logger.verbose("E:[jekyll-algolia] Raw error: #{error}")
-        Logger.verbose("E:[jekyll-algolia] Context: #{context}")
 
         identified_error = identify(error, context)
 
@@ -172,9 +171,27 @@ module Jekyll
         }
       end
 
+      # Public: Returns a string explaining which attributes are the largest in
+      # the record
+      #
+      # record - The record hash to analyze
+      #
+      # This will be used on the `record_too_big` error, to guide users in
+      # finding which record is causing trouble
+      def self.readable_largest_record_keys(record)
+        keys = Hash[record.map { |key, value| [key, value.to_s.length] }]
+        largest_keys = keys.sort_by { |_, value| value }.reverse[0..2]
+        output = []
+        largest_keys.each do |key, size|
+          size = Filesize.from("#{size} B").to_s('Kb')
+          output << "#{key} (#{size})"
+        end
+        output.join(', ')
+      end
+
       # Public: Check if the sent records are not too big
       #
-      # context[:records] - list of records to push
+      # context[:records] - list of records sent in the batch
       #
       # Records cannot weight more that 10Kb. If we're getting this error it
       # means that one of the records is too big, so we'll try to give
@@ -187,17 +204,25 @@ module Jekyll
 
         # Getting the record size
         size, = /.*size=(.*) bytes.*/.match(message).captures
-        size = Filesize.from("#{size} B").pretty
+        size = Filesize.from("#{size} B").to_s('Kb')
         object_id = details['objectID']
 
         # Getting record details
         record = Utils.find_by_key(context[:records], :objectID, object_id)
+        probable_wrong_keys = readable_largest_record_keys(record)
+
+        # Writing the full record to disk for inspection
+        record_log_path = Logger.write_to_file(
+          "jekyll-algolia-record-too-big-#{object_id}.log",
+          JSON.pretty_generate(record)
+        )
 
         {
           'object_id' => object_id,
           'object_title' => record[:title],
           'object_url' => record[:url],
-          'object_hint' => record[:content][0..100],
+          'probable_wrong_keys' => probable_wrong_keys,
+          'record_log_path' => record_log_path,
           'nodes_to_index' => Configurator.algolia('nodes_to_index'),
           'size' => size,
           'size_limit' => '10 Kb'

data/lib/jekyll/algolia/extractor.rb

@@ -11,7 +11,6 @@ module Jekyll
       # Public: Extract records from the file
       #
       # file - The Jekyll file to process
-      # TOTEST
       def self.run(file)
         # Getting all hierarchical nodes from the HTML input
         raw_records = extract_raw_records(file.content)

data/lib/jekyll/algolia/file_browser.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'algolia_html_extractor'
+require 'pathname'
 
 module Jekyll
   module Algolia
@@ -22,6 +23,35 @@ module Jekyll
         file.is_a?(Jekyll::StaticFile)
       end
 
+      # Public: Return the absolute path of a Jekyll file
+      #
+      # file - The Jekyll file to inspect
+      #
+      # Jekyll handles the .path property of some files as relative to the root
+      # (pages) or as an absolute paths (posts and static assets). We make sure
+      # we have a consistent way of accessing it
+      def self.absolute_path(file)
+        pathname = Pathname.new(file.path)
+        return pathname.cleanpath.to_s if pathname.absolute?
+
+        File.join(Configurator.get('source'), file.path)
+      end
+
+      # Public: Return the path of a Jekyll file relative to the Jekyll source
+      #
+      # file - The Jekyll file to inspect
+      #
+      # Jekyll handles the .path property of some files as relative to the root
+      # (pages) or as an absolute paths (posts and static assets). We make sure
+      # we have a consistent way of accessing it
+      def self.relative_path(file)
+        pathname = Pathname.new(file.path)
+        return file.path if pathname.relative?
+
+        jekyll_source = Pathname.new(Configurator.get('source'))
+        pathname.relative_path_from(jekyll_source).cleanpath.to_s
+      end
+
       # Public: Check if the file is a 404 error page
       #
       # file - The Jekyll file
@@ -86,16 +116,19 @@ module Jekyll
       # file - The Jekyll file
       def self.excluded_from_config?(file)
         excluded_patterns = Configurator.algolia('files_to_exclude')
-        excluded_files = []
+        jekyll_source = Configurator.get('source')
 
         # Transform the glob patterns into a real list of files
-        Dir.chdir(Configurator.get('source')) do
+        excluded_files = []
+        Dir.chdir(jekyll_source) do
           excluded_patterns.each do |pattern|
-            excluded_files += Dir.glob(pattern)
+            Dir.glob(pattern).each do |match|
+              excluded_files << File.expand_path(match)
+            end
           end
         end
 
-        excluded_files.include?(file.path)
+        excluded_files.include?(absolute_path(file))
       end
 
       # Public: Check if the file has been excluded by running a custom user
@@ -106,18 +139,6 @@ module Jekyll
         Hooks.should_be_excluded?(file.path)
       end
 
-      # Public: Return the path to the original file, relative from the Jekyll
-      # source
-      #
-      # file - The Jekyll file
-      #
-      # Pages have their .path property relative to the source, but collections
-      # (including posts) have an absolute file path.
-      def self.path_from_root(file)
-        source = Configurator.get('source')
-        file.path.gsub(%r{^#{source}/}, '')
-      end
-
       # Public: Check if the file should be indexed
       #
       # file - The Jekyll file
@@ -233,14 +254,32 @@ module Jekyll
         date.to_i
       end
 
+      # Public: Returns the raw excerpt of a file, directly as returned by
+      # Jekyll. Swallow any error that could occur when reading.
+      #
+      # file - The Jekyll file
+      #
+      # This might throw an exception if the excerpt is invalid. We also
+      # silence all logger output as Jekyll is quite verbose and will display
+      # the potential Liquid error in the terminal, even if we catch the actual
+      # error.
+      def self.excerpt_raw(file)
+        Logger.silent do
+          return file.data['excerpt'].to_s
+        end
+      rescue StandardError
+        return nil
+      end
+
       # Public: Returns the HTML version of the excerpt
       #
       # file - The Jekyll file
       #
       # Only collections (including posts) have an excerpt. Pages don't.
       def self.excerpt_html(file)
-        excerpt = file.data['excerpt']
+        excerpt = excerpt_raw(file)
         return nil if excerpt.nil?
+        return nil if excerpt.empty?
         excerpt.to_s.tr("\n", ' ').strip
       end
 
@@ -251,7 +290,6 @@ module Jekyll
       # Only collections (including posts) have an excerpt. Pages don't.
       def self.excerpt_text(file)
         html = excerpt_html(file)
-        return nil if html.nil?
         Utils.html_to_text(html)
       end
 

data/lib/jekyll/algolia/indexer.rb

@@ -151,7 +151,10 @@ module Jekyll
           begin
             ::Algolia.batch!(slice)
           rescue StandardError => error
-            ErrorHandler.stop(error)
+            records = slice.map do |record|
+              record[:body]
+            end
+            ErrorHandler.stop(error, records: records)
           end
         end
       end

data/lib/jekyll/algolia/logger.rb

@@ -4,6 +4,25 @@ module Jekyll
   module Algolia
     # Display helpful error messages
     module Logger
+      # Public: Silence all Jekyll log output in this block
+      # Usage:
+      #   Logger.silence do
+      #     # whatever Jekyll code here
+      #   end
+      #
+      # This is especially useful when Jekyll is too talkative about what is
+      # loggued. It works by redefining Jekyll.logger.write to a noop
+      # temporarily and re-attributing the original method once finished.
+      def self.silent
+        initial_method = Jekyll.logger.method(:write)
+        Utils.monkey_patch(Jekyll.logger, :write, proc { |*args| })
+        begin
+          yield
+        ensure
+          Utils.monkey_patch(Jekyll.logger, :write, initial_method)
+        end
+      end
+
       # Public: Displays a log line
       #
       # line - Line to display. Expected to be of the following format:
@@ -32,6 +51,16 @@ module Jekyll
         log(line)
       end
 
+      # Public: Write the specified content to a file in the source directory
+      #
+      # filename - the file basename
+      # content - the actual content of the file
+      def self.write_to_file(filename, content)
+        filepath = File.join(Configurator.get('source'), filename)
+        File.write(filepath, content)
+        filepath
+      end
+
       # Public: Displays a helpful error message for one of the knows errors
       #
       # message_id: A string identifying a know message

data/lib/jekyll/algolia/utils.rb

@@ -6,6 +6,19 @@ module Jekyll
   module Algolia
     # Generic language-wide utils
     module Utils
+      # Public: Allow redefining an instance method on the fly with a new one
+      #
+      # instance - The instance to overwrite
+      # method - The method symbol to overwrite
+      # block - The new block to use for replacing (as a proc)
+      #
+      # Solution found on
+      # https://stackoverflow.com/questions/803020/redefining-a-single-ruby-method-on-a-single-instance-with-a-lambda/16631789
+      def self.monkey_patch(instance, method, block)
+        metaclass = class << instance; self; end
+        metaclass.send(:define_method, method, block)
+      end
+
       # Public: Convert a hash with string keys to a hash with symbol keys
       #
       # hash - The input hash, with string keys
@@ -17,6 +30,7 @@ module Jekyll
       #
       # html - String representation of the HTML node
       def self.html_to_text(html)
+        return nil if html.nil?
         text = Nokogiri::HTML(html).text
         text.tr("\n", ' ').squeeze(' ').strip
       end

data/lib/jekyll/algolia/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module Algolia
-    VERSION = '1.1.0'
+    VERSION = '1.1.1'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-algolia
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Tim Carry
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-30 00:00:00.000000000 Z
+date: 2018-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: algolia_html_extractor
@@ -66,20 +66,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: jekyll-paginate
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.6'
+- !ruby/object:Gem::Dependency
+  name: jekyll-paginate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement