jekyll-algolia 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df27608a53e92a6a6045233960ebed4d0428d724
-  data.tar.gz: 7aa5442e7b35440a88c70a71336f7055bd7739a3
+  metadata.gz: 481b7a00fb77312b22d28dd2f36c1451716016ca
+  data.tar.gz: c9d0db5c134418357f510fd30085031b28a6ffca
 SHA512:
-  metadata.gz: cac19bf56a3777f5c6754e9b4309b22fe751c7076d54de4a91bd67e067c8a84f566542ad6b93b086a650cd844fa684f5fe9101605045428ded68c64a56370a80
-  data.tar.gz: 90b7b874c0a82d2b9e2d6015795c18ef9656ea41ab9f13b8c43de7309407bc023e9ee4a167343f4ea0f735303c192d38a444e2e6f32bb2e3ed77abda96451923
+  metadata.gz: 4cb7407675f31cdd3f11a4584df1f110934fab993fd60d98fe591dbfa82fd56a2773d3d7721dcc7a488c0c1977e67429226bdd500520a85c995f1c60936bfb36
+  data.tar.gz: 061740db0b6b2061eb0b16116f1b5c9a0425b191096030b5d99967cccd1b4fa611c78c5e9e4fb903d6ed067b558c623fdacf75dc0426694d3b8bbb15357bb1f6

data/README.md

@@ -45,12 +45,12 @@ Once this is done, download all dependencies with `bundle install`.
 You need to provide certain Algolia credentials for this plugin to *index* your
 site.
 
-*If you don't yet have an Algolia account, you can open a free [Community plan
-here][8]. Once signed in, you can get your
-credentials from [your dashboard][9].*
+*If you don't yet have an Algolia account, we suggest that you open a free
+[Community plan here][8]. You can find more information about the Algolia plans
+[in our FAQ][10].*
 
-Once you have your credentials, you should define your `application_id` inside
-your `_config.yml` file like this:
+Once signed in, you should get your application ID from [your dashboard][9] and
+define it inside your `_config.yml` file like this:
 
 ```yaml
 # _config.yml
@@ -72,18 +72,21 @@ Note that `ALGOLIA_API_KEY` should be set to your admin API key.
 
 ## More about the Community plan
 
-The Algolia [Community plan][10] lets you host up to 10k records and perform up to
-100k add/edit/delete operations per month (search operations are free). The plan
-is entirely free, with no time limit.
+The Algolia [Community plan][11] lets you host up to 10k records and perform up
+to 100k add/edit/delete operations per month (search operations are free). The
+plan is entirely free, with no time limit.
 
 What we ask in exchange is that you display a "Search by Algolia" logo next to
-your search results. Our [InstantSearch libraries][11] have a simple boolean
+your search results. Our [InstantSearch libraries][12] have a simple boolean
 option to toggle that on an off. If you want more flexibility, you can find
-all versions of our logo [here][12].
+all versions of our logo [here][13].
+
+If you need more information about the other Algolia plans, you can [check our
+FAQ][10].
 
 # Thanks
 
-Thanks to [Anatoliy Yastreb][13] for a [great tutorial][14] on creating Jekyll
+Thanks to [Anatoliy Yastreb][14] for a [great tutorial][15] on creating Jekyll
 plugins.
 
 
@@ -96,8 +99,9 @@ plugins.
 [7]: https://img.shields.io/badge/dynamic/json.svg?label=coverage%3Adevelop&colorB=&prefix=&suffix=%25&query=$.coverage_change&uri=https%3A%2F%2Fcoveralls.io%2Fgithub%2Falgolia%2Fjekyll-algolia.json%3Fbranch%3Ddevelop
 [8]: #more-about-the-community-plan
 [9]: https://www.algolia.com/api-keys
-[10]: https://www.algolia.com/users/sign_up/hacker
-[11]: https://community.algolia.com/instantsearch.js/
-[12]: https://www.algolia.com/press#resources
-[13]: https://github.com/ayastreb/
-[14]: https://ayastreb.me/writing-a-jekyll-plugin/
+[10]: https://community.algolia.com/jekyll-algolia/faq.html#how-many-records-will-the-plugin-need
+[11]: https://www.algolia.com/users/sign_up/hacker
+[12]: https://community.algolia.com/instantsearch.js/
+[13]: https://www.algolia.com/press#resources
+[14]: https://github.com/ayastreb/
+[15]: https://ayastreb.me/writing-a-jekyll-plugin/

data/lib/errors/record_too_big.txt

@@ -1,14 +1,10 @@
 E:[✗ Error] Record is too big
 E:
-E:The jekyll-algolia plugin could not push one of your records as it exceeds the {size_limit} size limit.
+E:The jekyll-algolia plugin detected that one of your records exceeds the {max_record_size} record size limit.
 W:
-W:The plugin will create one record for each element matching your `nodes_to_index` value (currently set to "{nodes_to_index}"). 
-W:Each record should not weight more than {size_limit}.
-W:One of your records weights {size} and has been rejected.
-W:
-W:objectID: {object_id}
 W:title:    {object_title}
 W:url:      {object_url}
+W:size:     {record_size}
 W:
 W:Most probable keys causing the issue:
 W:   {probable_wrong_keys}
@@ -18,5 +14,11 @@ W:   {record_log_path}
 W:
 I:This issue can be caused by malformed HTML preventing the parser to correctly grab the content of the nodes. Double check that the page actually renders correctly with a regular `jekyll build`.
 I:
-I:If you're having trouble solving this issue, feel free to file a bug on GitHub, ideally with a link to a repository where we can reproduce the issue.
+I:If you think this is an error and your current Algolia plan should allow you to push records bigger than {max_record_size}, you can change the `max_record_size` config option to increase the limit. Paid plans have a limit set to 20Kb, while free Community plans have it set to 10Kb.
+I:
+I:The following documentation might help you:
+I:   - https://community.algolia.com/jekyll-algolia/options.html#nodes-to-index
+I:   - https://community.algolia.com/jekyll-algolia/options.html#max-record-size
+I:
+I:If you're still having trouble solving this issue, feel free to file a bug on GitHub, ideally with a link to a repository where we can reproduce the issue.
 I:  https://github.com/algolia/jekyll-algolia/issues

data/lib/errors/record_too_big_api.txt

@@ -0,0 +1,10 @@
+E:[✗ Error] Record is too big
+E:
+E:The API rejected one of your records because it is too big ({record_size}).
+W:
+W:Make sure the value of `max_record_size` (currently set to "{max_record_size}") matches the maximum size allowed by your current Algolia plan.
+W:
+I:Free Community plans can push records up to 10Kb (`max_record_size: 10000`, this is the default value), while paid plans can go up to 20Kb (`max_record_size: 20000`).
+I:
+I:If you're still having trouble solving this issue, feel free to file a bug on GitHub, ideally with a link to a repository where we can reproduce the issue.
+I:  https://github.com/algolia/jekyll-algolia/issues

data/lib/jekyll/algolia/configurator.rb

@@ -14,6 +14,7 @@ module Jekyll
         'files_to_exclude' => nil,
         'nodes_to_index' => 'p',
         'indexing_batch_size' => 1000,
+        'max_record_size' => 10_000,
         'settings' => {
           # Searchable attributes
           'searchableAttributes' => %w[

data/lib/jekyll/algolia/error_handler.rb

@@ -47,7 +47,7 @@ module Jekyll
         known_errors = %w[
           unknown_application_id
           invalid_credentials
-          record_too_big
+          record_too_big_api
           too_many_records
           unknown_setting
           invalid_index_name
@@ -135,24 +135,6 @@ module Jekyll
         hash
       end
 
-      # Public: Returns a string explaining which attributes are the largest in
-      # the record
-      #
-      # record - The record hash to analyze
-      #
-      # This will be used on the `record_too_big` error, to guide users in
-      # finding which record is causing trouble
-      def self.readable_largest_record_keys(record)
-        keys = Hash[record.map { |key, value| [key, value.to_s.length] }]
-        largest_keys = keys.sort_by { |_, value| value }.reverse[0..2]
-        output = []
-        largest_keys.each do |key, size|
-          size = Filesize.from("#{size} B").to_s('Kb')
-          output << "#{key} (#{size})"
-        end
-        output.join(', ')
-      end
-
       # Public: Check if the application id is available
       #
       # _context - Not used
@@ -197,43 +179,24 @@ module Jekyll
       #
       # context[:records] - list of records sent in the batch
       #
-      # Records cannot weight more that 10Kb. If we're getting this error it
-      # means that one of the records is too big, so we'll try to give
-      # informations about it so the user can debug it.
-      def self.record_too_big?(error, context = {})
+      # One of the sent record is too big and has been rejected by the API. This
+      # should not happen as we proactively check for record size before pushing
+      # them. If it still happens it means that the value set in max_record_size
+      # is not matching the value in the plan.
+      def self.record_too_big_api?(error, _contex = {})
         details = error_hash(error.message)
         return false if details == false
 
         message = details['message']
         return false if message !~ /^Record .* is too big .*/
 
-        # Getting the record size
-        size, = /.*size=(.*) bytes.*/.match(message).captures
-        size = Filesize.from("#{size} B").to_s('Kb')
-        object_id = details['objectID']
-
-        # Finding the record in all the operations
-        operation = context[:operations].find do |o|
-          o[:action] == 'addObject' && o[:body][:objectID] == object_id
-        end
-        record = operation[:body]
-        probable_wrong_keys = readable_largest_record_keys(record)
-
-        # Writing the full record to disk for inspection
-        record_log_path = Logger.write_to_file(
-          "jekyll-algolia-record-too-big-#{object_id}.log",
-          JSON.pretty_generate(record)
-        )
+        record_size, = /.*size=(.*) bytes.*/.match(message).captures
+        record_size_readable = Filesize.from("#{record_size}B").to_s('Kb')
+        max_record_size = Configurator.algolia('max_record_size')
 
         {
-          'object_id' => object_id,
-          'object_title' => record[:title],
-          'object_url' => record[:url],
-          'probable_wrong_keys' => probable_wrong_keys,
-          'record_log_path' => record_log_path,
-          'nodes_to_index' => Configurator.algolia('nodes_to_index'),
-          'size' => size,
-          'size_limit' => '10 Kb'
+          'record_size' => record_size_readable,
+          'max_record_size' => max_record_size
         }
       end
 

data/lib/jekyll/algolia/overwrites/githubpages-configuration.rb

@@ -10,15 +10,23 @@ module GitHubPages
   # github-pages gem installed at the same time, those hooks will never be
   # executed.
   #
-  # Here, we overwrite the call to GitHubPages::Configuration.set that init the
-  # whole plugin to actually do nothing, hence disabling the plugin completely.
-  # https://github.com/github/pages-gem/blob/master/lib/github-pages.rb#L19
+  # The GitHub Pages gem prevent access to custom plugins by doing two things:
+  # - forcing safe mode
+  # - loading custom plugins from a random dir
+  #
+  # We cancel those by disabling safe mode and forcing back plugins to be read
+  # from ./_plugins.
   #
   # This file will only be loaded when running `jekyll algolia`, so it won't
   # interfere with the regular usage of `jekyll build`
   class Configuration
     class << self
-      def set(site); end
+      def set!(site)
+        config = effective_config(site.config)
+        config['safe'] = false
+        config['plugins_dir'] = '_plugins'
+        site.config = config
+      end
     end
   end
 end

data/lib/jekyll/algolia/overwrites/jekyll-algolia-site.rb

@@ -130,13 +130,15 @@ module Jekyll
         # Applying the user hook on the whole list of records
         records = Hooks.apply_all(records, self)
 
-        # Shrinking records to make them fit under 10Kb
+        # Shrinking records to force them to fit under the max record size
+        # limit, or displaying an error message if not possible
+        max_record_size = Configurator.algolia('max_record_size')
         # We take into account the objectID that will be added in the form of:
         # "objectID": "16cd998991cc40d92402b0b4e6c55e8a"
         object_id_attribute_length = 46
-        max_file_size = 10_000 - object_id_attribute_length
+        max_record_size -= object_id_attribute_length
         records.map! do |record|
-          Shrinker.fit_to_size(record, max_file_size)
+          Shrinker.fit_to_size(record, max_record_size)
         end
 
         # Adding a unique objectID to each record

data/lib/jekyll/algolia/shrinker.rb

@@ -25,7 +25,7 @@ module Jekyll
         return raw_record if size(raw_record) <= max_size
 
         # No excerpt, we can't shrink it
-        return raw_record unless raw_record.key?(:excerpt_html)
+        return stop_with_error(raw_record) unless raw_record.key?(:excerpt_html)
 
         record = raw_record.clone
 
@@ -41,10 +41,69 @@ module Jekyll
         return record if size(record) <= max_size
 
         # We remove the excerpts completely
-        record[:excerpt_text] = nil
-        record[:excerpt_html] = nil
+        record.delete(:excerpt_text)
+        record.delete(:excerpt_html)
+        return record if size(record) <= max_size
+
+        # Still too big, we fail
+        stop_with_error(record)
+      end
+
+      # Public: Stop the current indexing process and display details about the
+      # record that is too big to be pushed
+      #
+      # - record: The record causing the error
+      #
+      # This will display an error message and log the wrong record in a file in
+      # the source directory
+      def self.stop_with_error(record)
+        record_size = size(record)
+        record_size_readable = Filesize.from("#{record_size}B").to_s('Kb')
+        max_record_size = Configurator.algolia('max_record_size')
+        max_record_size_readable = Filesize
+                                   .from("#{max_record_size}B").to_s('Kb')
+
+        probable_wrong_keys = readable_largest_record_keys(record)
+
+        # Writing the full record to disk for inspection
+        record_log_path = Logger.write_to_file(
+          'jekyll-algolia-record-too-big.log',
+          JSON.pretty_generate(record)
+        )
+
+        details = {
+          'object_title' => record[:title],
+          'object_url' => record[:url],
+          'probable_wrong_keys' => probable_wrong_keys,
+          'record_log_path' => record_log_path,
+          'nodes_to_index' => Configurator.algolia('nodes_to_index'),
+          'record_size' => record_size_readable,
+          'max_record_size' => max_record_size_readable
+        }
+
+        Logger.known_message('record_too_big', details)
+
+        stop_process
+      end
+
+      # Public: Returns a string explaining which attributes are the largest in
+      # the record
+      #
+      # record - The record hash to analyze
+      def self.readable_largest_record_keys(record)
+        keys = Hash[record.map { |key, value| [key, value.to_s.length] }]
+        largest_keys = keys.sort_by { |_, value| value }.reverse[0..2]
+        output = []
+        largest_keys.each do |key, size|
+          size = Filesize.from("#{size} B").to_s('Kb')
+          output << "#{key} (#{size})"
+        end
+        output.join(', ')
+      end
 
-        record
+      # Public: Stop the current process
+      def self.stop_process
+        exit 1
       end
     end
   end

data/lib/jekyll/algolia/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module Algolia
-    VERSION = '1.4.0'
+    VERSION = '1.4.1'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-algolia
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Tim Carry
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-26 00:00:00.000000000 Z
+date: 2018-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: algolia_html_extractor
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.1
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.1
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: algoliasearch
   requirement: !ruby/object:Gem::Requirement
@@ -291,6 +291,7 @@ files:
 - lib/errors/missing_index_name.txt
 - lib/errors/no_records_found.txt
 - lib/errors/record_too_big.txt
+- lib/errors/record_too_big_api.txt
 - lib/errors/settings_manually_edited.txt
 - lib/errors/too_many_records.txt
 - lib/errors/unknown_application_id.txt