algoliasearch-jekyll 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 300e5c91ad0901840959d34b5cbb65d85f8508a8
+  data.tar.gz: fb8ec8f5a6a9afb49c81adf20c0b09ddbc4da15f
+SHA512:
+  metadata.gz: cee39f180e347887d52eaf144530aa146e057be93f395291d7f74e4e4ee608de2d8a804b7a6334b5d38247d5e4b1e5641fc192c7ae5b55a6f3dd85eb9852fde4
+  data.tar.gz: 2864b693fa1409dda9edfe64724c2e771ab5d9b6c778f8d1034dcd931471e5d02ce5c8bff2708db4711a5898bb266d3c6190428c2fab26d6e0576007973a997b

data/lib/algoliasearch-jekyll.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'awesome_print'
+
+require_relative './push.rb'
+
+# `jekyll algolia` main entry
+class AlgoliaSearchJekyll < Jekyll::Command
+  class << self
+    def init_with_program(prog)
+      prog.command(:algolia) do |command|
+        command.syntax 'algolia <command> [options]'
+        command.description 'Keep your content in sync with your Algolia index'
+
+        command.command(:push) do |subcommand|
+          subcommand.syntax 'push [options]'
+          subcommand.description 'Push your content to your index'
+
+          add_build_options(subcommand)
+
+          subcommand.action do |args, options|
+            @config = configuration_from_options(options)
+            AlgoliaSearchJekyllPush.process(args, options, @config)
+          end
+        end
+      end
+    end
+
+    # Allow a subset of the default `jekyll build` options
+    def add_build_options(command)
+      command.option 'config', '--config CONFIG_FILE[,CONFIG_FILE2,...]',
+                     Array, 'Custom configuration file'
+      command.option 'future', '--future', 'Index posts with a future date'
+      command.option 'limit_posts', '--limit_posts MAX_POSTS', Integer,
+                     'Limits the number of posts to parse and index'
+      command.option 'show_drafts', '-D', '--drafts',
+                     'Index posts in the _drafts folder'
+      command.option 'unpublished', '--unpublished',
+                     'Index posts that were marked as unpublished'
+    end
+
+    def api_key
+      return ENV['ALGOLIA_API_KEY'] if ENV['ALGOLIA_API_KEY']
+      key_file = File.join(@config['source'], '_algolia_api_key')
+
+      if File.exist?(key_file) && File.size(key_file) > 0
+        return File.open(key_file).read.strip
+      end
+      nil
+    end
+  end
+end

data/lib/push.rb

@@ -0,0 +1,226 @@
+require 'algoliasearch'
+require 'nokogiri'
+require 'json'
+
+# `jekyll algolia push` command
+class AlgoliaSearchJekyllPush < Jekyll::Command
+  class << self
+    def init_with_program(_prog)
+    end
+
+    def process(args = [], options = {}, config = {})
+      @args = args
+      @options = options
+      @config = config
+
+      index_name = args[0]
+
+      @config['algolia']['index_name'] = index_name if index_name
+      site = Jekyll::Site.new(@config)
+
+      # Instead of writing generated website to disk, we will push it to the
+      # index
+      def site.write
+        items = []
+        each_site_file do |file|
+          new_items = AlgoliaSearchJekyllPush.get_items_from_file(file)
+          next if new_items.nil?
+          items += new_items
+        end
+        AlgoliaSearchJekyllPush.push(items)
+      end
+
+      site.process
+    end
+
+    def markdown?(filename)
+      ext = File.extname(filename).delete('.')
+      @config['markdown_ext'].split(',').include?(ext)
+    end
+
+    def check_credentials(api_key, application_id, index_name)
+      unless api_key
+        Jekyll.logger.error 'Algolia Error: No API key defined'
+        Jekyll.logger.warn '  You have two ways to configure your API key:'
+        Jekyll.logger.warn '    - The ALGOLIA_API_KEY environment variable'
+        Jekyll.logger.warn '    - A file named ./_algolia_api_key in your '\
+                           'source folder'
+        exit 1
+      end
+
+      unless application_id
+        Jekyll.logger.error 'Algolia Error: No application ID defined'
+        Jekyll.logger.warn '  Please set your application id in the '\
+                           '_config.yml file, like so:'
+        puts ''
+        # The spaces are needed otherwise the text is centered
+        Jekyll.logger.warn '  algolia:         '
+        Jekyll.logger.warn '    application_id: \'{your_application_id}\''
+        puts ''
+        Jekyll.logger.warn '  Your application ID can be found in your algolia'\
+                           ' dashboard'
+        Jekyll.logger.warn '    https://www.algolia.com/licensing'
+        exit 1
+      end
+
+      unless index_name
+        Jekyll.logger.error 'Algolia Error: No index name defined'
+        Jekyll.logger.warn '  Please set your index name in the _config.yml'\
+                           ' file, like so:'
+        puts ''
+        # The spaces are needed otherwise the text is centered
+        Jekyll.logger.warn '  algolia:         '
+        Jekyll.logger.warn '    index_name: \'{your_index_name}\''
+        puts ''
+        Jekyll.logger.warn '  You can edit your indices in your dashboard'
+        Jekyll.logger.warn '    https://www.algolia.com/explorer'
+        exit 1
+      end
+      true
+    end
+
+    def configure_index(index)
+      index.set_settings(
+        attributeForDistinct: 'parent_id',
+        attributesForFaceting: %w(tags type),
+        attributesToHighlight: %w(title content),
+        attributesToIndex: %w(title h1 h2 h3 h4 h5 h6 content tags),
+        attributesToRetrieve: %w(title posted_at content url css_selector),
+        customRanking: ['desc(posted_at)', 'desc(title_weight)'],
+        distinct: true,
+        highlightPreTag: '<span class="algolia__result-highlight">',
+        highlightPostTag: '</span>'
+      )
+    end
+
+    def push(items)
+      api_key = AlgoliaSearchJekyll.api_key
+      application_id = @config['algolia']['application_id']
+      index_name = @config['algolia']['index_name']
+      check_credentials(api_key, application_id, index_name)
+
+      Algolia.init(application_id: application_id, api_key: api_key)
+      index = Algolia::Index.new(index_name)
+      configure_index(index)
+      index.clear_index
+
+      items.each_slice(1000) do |batch|
+        Jekyll.logger.info "Indexing #{batch.size} items"
+        begin
+          index.add_objects(batch)
+        rescue StandardError => error
+          Jekyll.logger.error 'Algolia Error: HTTP Error'
+          Jekyll.logger.warn error.message
+          exit 1
+        end
+      end
+
+      Jekyll.logger.info "Indexing of #{items.size} items " \
+                         "in #{index_name} done."
+    end
+
+    def get_items_from_file(file)
+      is_page = file.is_a?(Jekyll::Page)
+      is_post = file.is_a?(Jekyll::Post)
+
+      # We only index posts, and markdown pages
+      return nil unless is_page || is_post
+      return nil if is_page && !markdown?(file.path)
+
+      html = file.content.gsub("\n", ' ')
+
+      if is_post
+        tags = get_tags_from_post(file)
+        base_data = {
+          type: 'post',
+          parent_id: file.id,
+          url: file.url,
+          title: file.title,
+          tags: tags,
+          slug: file.slug,
+          posted_at: file.date.to_time.to_i
+        }
+      else
+        base_data = {
+          type: 'page',
+          parent_id: file.basename,
+          url: file.url,
+          title: file['title'],
+          slug: file.basename
+        }
+      end
+
+      get_paragraphs_from_html(html, base_data)
+    end
+
+    # Get a list of tags from a post. Handle both classic string tags or
+    # extended object tags
+    def get_tags_from_post(post)
+      tags = post.tags
+      return [] if tags.is_a?(Array) || tags.nil?
+      tags.map! { |tag| tag.to_s.gsub(',', '') }
+    end
+
+    def get_previous_hx(node, memo = { level: 7 })
+      previous = node.previous_sibling
+      # Stop if no previous element
+      unless previous
+        memo.delete(:level)
+        return memo
+      end
+
+      # Skip non-html elements
+      return get_previous_hx(previous, memo) unless previous.element?
+
+      # Skip non-title elements
+      tag_name = previous.name
+      possible_title_elements = %w(h1 h2 h3 h4 h5 h6)
+      unless possible_title_elements.include?(tag_name)
+        return get_previous_hx(previous, memo)
+      end
+
+      # Skip if item already as title of a higher level
+      title_level = tag_name.gsub('h', '').to_i
+      return get_previous_hx(previous, memo) if title_level >= memo[:level]
+      memo[:level] = title_level
+
+      # Add to the memo and continue
+      memo[tag_name.to_sym] = previous.text
+      get_previous_hx(previous, memo)
+    end
+
+    # Get a custom value representing the number of word occurence from the
+    # titles into the content
+    def get_title_weight(content, item)
+      # Get list of words
+      words = %i(title h1 h2 h3 h4 h5 h6)
+              .select { |title| item.key?(title) }
+              .map { |title| item[title].split(/\W+/) }
+              .flatten
+              .compact
+              .uniq
+      # Count how many words are in the text
+      weight = 0
+      words.each { |word| weight += 1 if content.include?(word) }
+      weight
+    end
+
+    # Will get a unique css selector for the node
+    def get_css_selector(node)
+      node.css_path.gsub('html > body > ', '')
+    end
+
+    def get_paragraphs_from_html(html, base_data)
+      doc = Nokogiri::HTML(html)
+      doc.css('p').map.with_index do |p, index|
+        new_item = base_data.clone
+        new_item.merge!(get_previous_hx(p))
+        new_item[:objectID] = "#{new_item[:parent_id]}_#{index}"
+        new_item[:css_selector] = get_css_selector(p)
+        new_item[:content] = p.to_s
+        new_item[:title_weight] = get_title_weight(p.text, new_item)
+        new_item
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: algoliasearch-jekyll
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Tim Carry
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: algoliasearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+description: Index all your pages and posts to an Algolia index with `jekyll algolia
+  index`
+email: tim@pixelastic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/algoliasearch-jekyll.rb
+- lib/push.rb
+homepage: https://github.com/algolia/algoliasearch-jekyll
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: AlgoliaSearch for Jekyll
+test_files: []