algoliasearch-jekyll 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1b192525e42f3f0b61d0587ab11fdef936acc10
-  data.tar.gz: b8557b3ae3e1df1d6e3561c3614a8c4e5e3dc23e
+  metadata.gz: f1af115b167749491e0cfc3fbdfca8f7cbd0bfb1
+  data.tar.gz: 6cdc40cf3148a33400178ba9bb9076095adebd31
 SHA512:
-  metadata.gz: b52e88d691848055afbe673401adbef36faf32ad3de1c62985577e0790fd6f97eb4ad8dd0e4f717c6c43b38b5c41a194652e254ed91fdc397931f413fcab3670
-  data.tar.gz: 356ea6edf1d3fdc9ff1b1792294afaa966661ab3c50e3a6eee0d7567e381f742df423b3f6349eb7761393385635c9599144e1e9a0ba2c4c219580790e2eea7f5
+  metadata.gz: 371291f704b4029819eb5dbb59de2e3ac2ac90c8973ae678fb1890824f3c9b4470782b1bfe8b7cbb528a374e80eeb4e274440c53a5561efc5bb3d703d4e19ead
+  data.tar.gz: c508df8e04d78ae5db324ef678cb27ae46f5e067030a45a373dc399f25eca244263ba3bddf998e35b965fed31fb374d1e565014e6fe7a06631f4e4831811b097

data/lib/algoliasearch-jekyll.rb

@@ -21,7 +21,7 @@ class AlgoliaSearchJekyll < Jekyll::Command
 
           subcommand.action do |args, options|
             @config = configuration_from_options(options)
-            AlgoliaSearchJekyllPush.process(args, options, @config)
+            AlgoliaSearchJekyllPush.init_options(args, options, @config).process
           end
         end
       end
@@ -39,15 +39,5 @@ class AlgoliaSearchJekyll < Jekyll::Command
       command.option 'unpublished', '--unpublished',
                      'Index posts that were marked as unpublished'
     end
-
-    def api_key
-      return ENV['ALGOLIA_API_KEY'] if ENV['ALGOLIA_API_KEY']
-      key_file = File.join(@config['source'], '_algolia_api_key')
-
-      if File.exist?(key_file) && File.size(key_file) > 0
-        return File.open(key_file).read.strip
-      end
-      nil
-    end
   end
 end

data/lib/push.rb

@@ -1,52 +1,92 @@
 require 'algoliasearch'
 require 'nokogiri'
 require 'json'
+require_relative './record_extractor.rb'
 
 # `jekyll algolia push` command
 class AlgoliaSearchJekyllPush < Jekyll::Command
   class << self
+    attr_accessor :options, :config
+
     def init_with_program(_prog)
     end
 
-    def process(args = [], options = {}, config = {})
+    # Init the command with options passed on the command line
+    # `jekyll algolia push ARG1 ARG2 --OPTION_NAME1 OPTION_VALUE1`
+    # config comes from _config.yml
+    def init_options(args = [], options = {}, config = {})
+      args = [] unless args
       @args = args
       @options = options
       @config = config
 
+      # Allow for passing index name on the command line
       index_name = args[0]
-
       @config['algolia']['index_name'] = index_name if index_name
+      self
+    end
+
+    # Check if the specified file should be indexed (we exclude static files,
+    # robots.txt and custom defined exclusions).
+    def indexable?(file)
+      return false if file.is_a?(Jekyll::StaticFile)
+
+      # Keep only markdown and html files
+      allowed_extensions = %w(html)
+      if @config['markdown_ext']
+        allowed_extensions += @config['markdown_ext'].split(',')
+      end
+      current_extension = File.extname(file.name)[1..-1]
+      return false unless allowed_extensions.include?(current_extension)
+
+      # Exclude files manually excluded from config
+      excluded_files = @config['algolia']['excluded_files']
+      unless excluded_files.nil?
+        return false if excluded_files.include?(file.name)
+      end
+
+      true
+    end
+
+    # Run the default `jekyll build` command but overwrite the actual "write
+    # files on disk" part to instead push data to Algolia
+    def process
       site = Jekyll::Site.new(@config)
 
-      # Instead of writing generated website to disk, we will push it to the
-      # index
       def site.write
         items = []
         each_site_file do |file|
-          new_items = AlgoliaSearchJekyllPush.get_items_from_file(file)
+          next unless AlgoliaSearchJekyllPush.indexable?(file)
+
+          new_items = AlgoliaSearchRecordExtractor.new(file).extract
           next if new_items.nil?
           items += new_items
         end
         AlgoliaSearchJekyllPush.push(items)
       end
 
+      # This will call the build command by default, which will in turn call our
+      # custom .write method
       site.process
     end
 
-    def parseable?(file)
-      ext = file.ext.delete('.')
-      # Allow markdown and html pages
-      return true if @config['markdown_ext'].split(',').include?(ext)
-      return false unless ext == 'html'
-      return false unless file['title']
-      true
-    end
+    # Read the API key either from ENV or from an _algolia_api_key file in
+    # source folder
+    def api_key
+      # First read in ENV
+      return ENV['ALGOLIA_API_KEY'] if ENV['ALGOLIA_API_KEY']
 
-    def excluded_file?(file)
-      @config['algolia']['excluded_files'].include?(file.name)
+      # Otherwise from file in source directory
+      key_file = File.join(@config['source'], '_algolia_api_key')
+      if File.exist?(key_file) && File.size(key_file) > 0
+        return File.open(key_file).read.strip
+      end
+      nil
     end
 
-    def check_credentials(api_key, application_id, index_name)
+    # Check that all credentials are present, and stop with a helpfull message
+    # if not
+    def check_credentials
       unless api_key
         Jekyll.logger.error 'Algolia Error: No API key defined'
         Jekyll.logger.warn '  You have two ways to configure your API key:'
@@ -56,30 +96,30 @@ class AlgoliaSearchJekyllPush < Jekyll::Command
         exit 1
       end
 
-      unless application_id
+      unless @config['algolia']['application_id']
         Jekyll.logger.error 'Algolia Error: No application ID defined'
         Jekyll.logger.warn '  Please set your application id in the '\
                            '_config.yml file, like so:'
-        puts ''
+        Jekyll.logger.warn ''
         # The spaces are needed otherwise the text is centered
         Jekyll.logger.warn '  algolia:         '
         Jekyll.logger.warn '    application_id: \'{your_application_id}\''
-        puts ''
+        Jekyll.logger.warn ''
         Jekyll.logger.warn '  Your application ID can be found in your algolia'\
                            ' dashboard'
         Jekyll.logger.warn '    https://www.algolia.com/licensing'
         exit 1
       end
 
-      unless index_name
+      unless @config['algolia']['index_name']
         Jekyll.logger.error 'Algolia Error: No index name defined'
         Jekyll.logger.warn '  Please set your index name in the _config.yml'\
                            ' file, like so:'
-        puts ''
+        Jekyll.logger.warn ''
         # The spaces are needed otherwise the text is centered
         Jekyll.logger.warn '  algolia:         '
         Jekyll.logger.warn '    index_name: \'{your_index_name}\''
-        puts ''
+        Jekyll.logger.warn ''
         Jekyll.logger.warn '  You can edit your indices in your dashboard'
         Jekyll.logger.warn '    https://www.algolia.com/explorer'
         exit 1
@@ -87,10 +127,13 @@ class AlgoliaSearchJekyllPush < Jekyll::Command
       true
     end
 
+    # Get index settings
     def configure_index(index)
-      default_settings = {
-        attributeForDistinct: 'parent_id',
-        attributesForFaceting: %w(tags type),
+      settings = {
+        typoTolerance: true,
+        distinct: true,
+        attributeForDistinct: 'title',
+        attributesForFaceting: %w(tags type title),
         attributesToIndex: %w(
           title h1 h2 h3 h4 h5 h6
           unordered(text)
@@ -98,143 +141,39 @@ class AlgoliaSearchJekyllPush < Jekyll::Command
         ),
         attributesToRetrieve: %w(
           title h1 h2 h3 h4 h5 h6
-          posted_at
-          content
-          text
           url
+          tag_name
+          raw_html
+          text
+          posted_at
           css_selector
+          css_selector_parent
         ),
         customRanking: ['desc(posted_at)', 'desc(title_weight)'],
-        distinct: true,
         highlightPreTag: '<span class="algolia__result-highlight">',
         highlightPostTag: '</span>'
       }
-      custom_settings = {}
-      @config['algolia']['settings'].each do |key, value|
-        custom_settings[key.to_sym] = value
-      end
-      settings = default_settings.merge(custom_settings)
-
-      index.set_settings(settings)
-    end
-
-    def get_items_from_file(file)
-      is_page = file.is_a?(Jekyll::Page)
-      is_post = file.is_a?(Jekyll::Post)
-
-      # We only index posts, and markdown pages
-      return nil unless is_page || is_post
-      return nil if is_page && !parseable?(file)
-      return nil if excluded_file?(file)
-
-      html = file.content.gsub("\n", ' ')
-
-      if is_post
-        tags = get_tags_from_post(file)
-        base_data = {
-          type: 'post',
-          parent_id: file.id,
-          url: file.url,
-          title: file.title,
-          tags: tags,
-          slug: file.slug,
-          posted_at: file.date.to_time.to_i
-        }
-      else
-        base_data = {
-          type: 'page',
-          parent_id: file.basename,
-          url: file.url,
-          title: file['title'],
-          slug: file.basename
-        }
-      end
-
-      get_paragraphs_from_html(html, base_data)
-    end
-
-    # Get a list of tags from a post. Handle both classic string tags or
-    # extended object tags
-    def get_tags_from_post(post)
-      tags = post.tags
-      return [] if tags.is_a?(Array) || tags.nil?
-      tags.map! { |tag| tag.to_s.gsub(',', '') }
-    end
-
-    # Get the list of headings (h1, h2, etc) above the specified node
-    def get_previous_hx(node, memo = { level: 7 })
-      previous = node.previous_sibling
-      # Stop if no previous element
-      unless previous
-        memo.delete(:level)
-        return memo
-      end
-
-      # Skip non-html elements
-      return get_previous_hx(previous, memo) unless previous.element?
 
-      # Skip non-title elements
-      tag_name = previous.name
-      possible_title_elements = %w(h1 h2 h3 h4 h5 h6)
-      unless possible_title_elements.include?(tag_name)
-        return get_previous_hx(previous, memo)
+      # Merge default settings with user custom ones
+      if @config['algolia'].key?('settings')
+        custom_settings = {}
+        @config['algolia']['settings'].each do |key, value|
+          custom_settings[key.to_sym] = value
+        end
+        settings.merge!(custom_settings)
       end
 
-      # Skip if item already as title of a higher level
-      title_level = tag_name.gsub('h', '').to_i
-      return get_previous_hx(previous, memo) if title_level >= memo[:level]
-      memo[:level] = title_level
-
-      # Add to the memo and continue
-      memo[tag_name.to_sym] = previous.content
-      get_previous_hx(previous, memo)
-    end
-
-    # Get a custom value representing the number of word occurence from the
-    # titles into the content
-    def get_title_weight(content, item)
-      # Get list of words
-      words = %i(title h1 h2 h3 h4 h5 h6)
-              .select { |title| item.key?(title) }
-              .map { |title| item[title].split(/\W+/) }
-              .flatten
-              .compact
-              .uniq
-      # Count how many words are in the text
-      weight = 0
-      words.each { |word| weight += 1 if content.include?(word) }
-      weight
-    end
-
-    # Will get a unique css selector for the node
-    def get_css_selector(node)
-      node.css_path.gsub('html > body > ', '')
-    end
-
-    # Get a list of items representing the different paragraphs
-    def get_paragraphs_from_html(html, base_data)
-      doc = Nokogiri::HTML(html)
-      paragraphs = doc.css('p').map.with_index do |p, index|
-        next unless p.text.size > 0
-        new_item = base_data.clone
-        new_item.merge!(get_previous_hx(p))
-        new_item[:objectID] = "#{new_item[:parent_id]}_#{index}"
-        new_item[:css_selector] = get_css_selector(p)
-        new_item[:raw_html] = p.to_s
-        new_item[:text] = p.content
-        new_item[:title_weight] = get_title_weight(p.text, new_item)
-        new_item
-      end
-      paragraphs.compact
+      index.set_settings(settings)
     end
 
     def push(items)
-      api_key = AlgoliaSearchJekyll.api_key
-      application_id = @config['algolia']['application_id']
-      index_name = @config['algolia']['index_name']
-      check_credentials(api_key, application_id, index_name)
+      check_credentials
 
-      Algolia.init(application_id: application_id, api_key: api_key)
+      index_name = @config['algolia']['index_name']
+      Algolia.init(
+        application_id: @config['algolia']['application_id'],
+        api_key: api_key
+      )
       index = Algolia::Index.new(index_name)
       configure_index(index)
       index.clear_index

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: algoliasearch-jekyll
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Tim Carry
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-30 00:00:00.000000000 Z
+date: 2015-07-03 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: jekyll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.6'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.4'
 description: Index all your pages and posts to an Algolia index with `jekyll algolia
-  index`
+  push`
 email: tim@pixelastic.com
 executables: []
 extensions: []