indexers 4.1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 596797c42610ddbebfd01c173d2d386d6c2d5cd8
+  data.tar.gz: 02a005767c01ef463b72c99343234e6c5a16145f
+SHA512:
+  metadata.gz: 595bfc3426cfd08b942ad4ba599d2eefdaf13af30c6312e7c25113aae37a2b2832a71ad733686793bf21da2ff8cd25c4229f3947c8d78000bf5c86fa5c222bbc
+  data.tar.gz: 9a52c3bfcc3388583988ea9d245a515ce28fd8e0a810252627eebf883450d113660cb101e1aa1462742756516b3c77d41b38aa3a538129c11e56c463dfb671bb

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 Mathías Montossi
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,313 @@
+[![Gem Version](https://badge.fury.io/rb/indexers.svg)](http://badge.fury.io/rb/indexers)
+[![Code Climate](https://codeclimate.com/github/mmontossi/indexers/badges/gpa.svg)](https://codeclimate.com/github/mmontossi/indexers)
+[![Build Status](https://travis-ci.org/mmontossi/indexers.svg)](https://travis-ci.org/mmontossi/indexers)
+[![Dependency Status](https://gemnasium.com/mmontossi/indexers.svg)](https://gemnasium.com/mmontossi/indexers)
+
+# Indexers
+
+Dsl to delegate searches to elasticsearch in rails.
+
+## Why
+
+I did this gem to:
+
+- Gain control of the queries without losing simplicity.
+- Have out of the box integration with activerecord and pagers.
+- Deal with the just in time nature of elasticsearch.
+- Integrate activerecord includes on it.
+- Have a convention of how to use suggestions.
+
+## Install
+
+Put this line in your Gemfile:
+```ruby
+gem 'indexers'
+```
+
+Then bundle:
+```
+$ bundle
+```
+
+To install Redis you can use homebrew:
+```
+$ brew install elasticsearch24
+```
+
+NOTE: This gem is tested agains version 2.4.
+
+## Configuration
+
+Generate the configuration file:
+```
+$ bundle exec rails g indexers:install
+```
+
+Set the global settings:
+```ruby
+Indexers.configure do |config|
+
+  config.mappings do
+    name do
+      type 'string'
+      fields do
+        raw do
+          type 'string'
+          index 'not_analyzed'
+        end
+      end
+    end
+    category type: 'string'
+    price type: 'long'
+    currency type: 'string'
+    product_suggestions do
+      type 'completion'
+      analyzer 'simple'
+    end
+  end
+
+end
+```
+
+If you need to personalize the analysis, you can it here:
+
+```ruby
+Indexers.configure do |config|
+
+  config.analysis do
+    filter do
+      ngram do
+        type 'nGram'
+        min_gram 2
+        max_gram 20
+      end
+    end
+    analyzer do
+      ngram do
+        type 'custom'
+        tokenizer 'standard'
+        filter %w(lowercase ngram)
+      end
+    end
+  end
+
+end
+```
+
+NOTE: You may want to personalize the generated config/elasticsearch.yml.
+
+### Definitions
+
+Generate an index:
+```
+$ bundle exec rails g indexers:indexer products
+```
+
+Define the mappings, serialization and search in the index:
+```ruby
+Indexers.define :product do
+
+  mappings do
+    properties :name, :category, :price, :product_suggestions
+  end
+
+  serialize do |record|
+    extract record, :name, :category, :price
+    product_suggestions do
+      input [record.name, transliterate(record.name)].uniq
+      output record.name
+    end
+  end
+
+  search do |*args|
+    options = args.extract_options!
+    term = args.first
+    query do
+      if term.present?
+        multi_match do
+          query term
+          type 'phrase_prefix'
+          fields %w(name category)
+        end
+      else
+        match_all
+      end
+    end
+  end
+
+end
+```
+
+### Traits
+
+You can dry complex searches or serializations using traits:
+```ruby
+Indexers.define :product do
+
+  search do |*args|
+    options = args.extract_options!
+    shop = options[:shop]
+    term = args.first
+    query do
+      filtered do
+        traits :shop
+        query do
+          if term.present?
+            multi_match do
+              query term
+              type 'phrase_prefix'
+              fields %w(name category)
+            end
+          else
+            match_all
+          end
+        end
+      end
+    end
+  end
+
+  trait :shop do
+    filter do
+      bool do
+        must do
+          if shop
+            term do
+              _parent shop.id
+            end
+          end
+        end
+      end
+    end
+  end
+
+end
+```
+
+NOTE: The binding is persisted, there is no need to redefine variables.
+
+### Indexing
+
+The index will be updated every time a record is created, updated or destroyed:
+```ruby
+product = Product.create(name: 'Les Paul', category: 'Gibson')
+```
+
+You can force this actions manually with:
+```ruby
+product.index
+product.reindex
+product.unindex
+```
+
+### Rake tasks
+
+At any time you can build/rebuild your indexers using:
+```
+$ bundle exec rake indexers:index
+$ bundle exec rake indexers:reindex
+$ bundle exec rake indexers:unindex
+```
+
+### Search
+
+Use the included search method in the model:
+```ruby
+products = Product.search('Les Paul')
+```
+
+The result can be used as a collection in views:
+```erb
+<%= render products %>
+```
+
+### Includes
+
+Similar to using activerecod:
+```ruby
+Product.search.includes(:shop)
+```
+
+### Pagination
+
+Works the same as [pagers gem](https://github.com/mmontossi/pagers):
+```ruby
+Products.search.page(1, padding: 4, length: 30)
+```
+
+You can force a record to be part of the results by id:
+```ruby
+Products.search.page(1, with: 4)
+```
+
+Or the opposite:
+```ruby
+Products.search.page(4, without: 4)
+```
+
+And you can send the collection directly to the view helper:
+```erb
+<%= paginate products %>
+```
+
+### Order
+
+Same as using activerecord:
+```ruby
+Product.search.order(name: :asc)
+```
+
+You can use a computed sort by declare it in the configuration:
+```ruby
+Indexers.configure do |config|
+
+  config.computed_sort :price do |direction|
+    type 'number'
+    script do
+      inline "if (_source.currency == 'UYU') { doc['price'].value * 30 }"
+    end
+    order direction
+  end
+
+end
+```
+
+### Suggestions
+
+You need to first define the logic in the configuration:
+```ruby
+Indexers.configure do |config|
+
+  config.suggestions do |name, term, options={}|
+    type = name.to_s.singularize
+    text (term || '')
+    completion do
+      field "#{type}_suggestions"
+    end
+  end
+
+end
+```
+
+Then you can get suggestions using the suggest method:
+```ruby
+Indexers.suggest :product, 'gibson'
+```
+
+The result is an array of hashes with a text property:
+```ruby
+[{ text: 'Les Paul' }, ...]
+```
+
+## Contributing
+
+Any issue, pull request, comment of any kind is more than welcome!
+
+I will mainly ensure compatibility to PostgreSQL, AWS, Redis, Elasticsearch, FreeBSD and Memcached. 
+
+## Credits
+
+This gem is maintained and funded by [mmontossi](https://github.com/mmontossi).
+
+## License
+
+It is free software, and may be redistributed under the terms specified in the MIT-LICENSE file.

data/Rakefile

@@ -0,0 +1,19 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+  t.warning = false
+end
+
+task default: :test

data/lib/generators/indexers/indexer/indexer_generator.rb

@@ -0,0 +1,23 @@
+require 'rails/generators'
+
+module Indexers
+  module Generators
+    class IndexerGenerator < Rails::Generators::NamedBase
+
+      source_root File.expand_path('../templates', __FILE__)
+
+      def create_index_file
+        template 'indexer.rb', File.join('app/indexers', class_path, "#{file_name}_indexer.rb")
+      end
+
+      private
+
+      def class_name_option
+        if class_path.any?
+          ", class_name: '#{class_name}'"
+        end
+      end
+
+    end
+  end
+end

data/lib/generators/indexers/indexer/templates/indexer.rb

@@ -0,0 +1,12 @@
+Indexers.define :<%= singular_table_name %><%= class_name_option %> do
+
+  mappings do
+  end
+
+  serialize do |record|
+  end
+
+  search do |*args|
+  end
+
+end

data/lib/generators/indexers/install/install_generator.rb

@@ -0,0 +1,19 @@
+require 'rails/generators'
+
+module Indexers
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+
+      source_root File.expand_path('../templates', __FILE__)
+
+      def create_initializer_file
+        copy_file 'initializer.rb', 'config/initializers/indexers.rb'
+      end
+
+      def create_configuration_file
+        copy_file 'configuration.yml', 'config/elasticsearch.yml'
+      end
+
+    end
+  end
+end

data/lib/generators/indexers/install/templates/configuration.yml

@@ -0,0 +1,15 @@
+default: &default
+  hosts: localhost
+  port: 9200
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default
+
+staging:
+  <<: *default

data/lib/generators/indexers/install/templates/initializer.rb

@@ -0,0 +1,6 @@
+Indexers.configure do |config|
+
+  config.mappings do
+  end
+
+end

data/lib/indexers/collection.rb

@@ -0,0 +1,181 @@
+module Indexers
+  class Collection
+    include Enumerable
+
+    attr_reader :indexer, :args, :options
+
+    delegate :model, to: :indexer
+    delegate :model_name, to: :model
+    delegate :each, :map, :size, :length, :count, :[], :to_a, to: :records
+
+    alias_method :to_ary, :to_a
+
+    def initialize(indexer, args, options)
+      @loaded = false
+      @indexer = indexer
+      @args = args
+      @options = options
+    end
+
+    def includes(*args)
+      chain includes: args
+    end
+
+    def page(number, options={})
+      length = page_option(options, :length, 10)
+      padding = page_option(options, :padding, 0)
+      current_page = [number.to_i, 1].max
+      values = Module.new do
+        define_method :page_length do
+          length
+        end
+        define_method :padding do
+          padding
+        end
+        define_method :current_page do
+          current_page
+        end
+      end
+      overrides = {
+        from: ((length * (current_page - 1)) + padding),
+        size: length
+      }
+      %i(with without).each do |name|
+        if options.has_key?(name)
+          overrides[name] = options[name]
+        end
+      end
+      chain Pagination, values, overrides
+    end
+
+    def order(options)
+      mappings = Indexers.configuration.mappings
+      values = []
+      options.each do |property, direction|
+        if block = Indexers.computed_sorts.find(property)
+          values << { _script: Dsl::Api.new(direction, &block).to_h }
+        elsif property == :id
+          values << { _uid: { order: direction } }
+        elsif mappings.has_key?(property) && mappings[property][:type] == 'string'
+          values << { "#{property}.raw" => { order: direction } }
+        end
+      end
+      if values.any?
+        chain sort: values
+      else
+        chain
+      end
+    end
+
+    def response
+      if @loaded == true
+        @response
+      else
+        @loaded = true
+        @response = indexer.search(query)
+      end
+    end
+
+    def query
+      @query ||= begin
+        pagination = options.slice(:from, :size, :sort)
+        without_ids = fetch_ids(options[:without])
+        body = Dsl::Search.new(indexer, args.append(options), &indexer.options[:search]).to_h[:query]
+        request = Dsl::Search.new do
+          if without_ids.any?
+            query do
+              filtered do
+                filter do
+                  bool do
+                    must_not do
+                      without_ids.each do |id|
+                        term do
+                          _id id
+                        end
+                      end
+                    end
+                  end
+                end
+                query body
+              end
+            end
+          else
+            query body
+          end
+          %i(from size).each do |name|
+            if pagination.has_key?(name)
+              send name, pagination[name]
+            end
+          end
+          if pagination.has_key?(:sort)
+            sort pagination[:sort]
+          else
+            sort do
+              _uid do
+                order 'desc'
+              end
+            end
+          end
+        end
+        request.to_h
+      end
+    end
+
+    private
+
+    def records
+      @records ||= begin
+        hit_ids = response['hits']['hits'].map{ |hit| hit['_id'].to_i }
+        missing_ids = (fetch_ids(options[:with]) - hit_ids)
+        if missing_ids.any?
+          last_index = -(missing_ids.length + 1)
+          ids = (missing_ids.sort.reverse + hit_ids.to(last_index))
+        else
+          ids = hit_ids
+        end
+        includes = options.fetch(:includes, [])
+        indexer.model.includes(includes).where(id: ids).sort do |a,b|
+          ids.index(a.id) <=> ids.index(b.id)
+        end
+      end
+    end
+
+    def page_option(source, name, default)
+      source[name] || begin
+        if Rails.configuration.cache_classes == false
+          Rails.application.eager_load!
+        end
+        if defined?(Pagers)
+          Pagers.configuration.send name
+        else
+          default
+        end
+      end
+    end
+
+    def fetch_ids(source)
+      case source
+      when Fixnum,String
+        [source.to_i]
+      when ActiveRecord::Base
+        [source.id]
+      when ActiveRecord::Relation
+        source.ids
+      when Array
+        source.map{ |value| fetch_ids(value) }.flatten
+      else
+        []
+      end
+    end
+
+    def chain(*extensions)
+      overrides = extensions.extract_options!
+      collection = Collection.new(indexer, args, options.merge(overrides))
+      extensions.each do |extension|
+        collection.extend extension
+      end
+      collection
+    end
+
+  end
+end

data/lib/indexers/computed_sorts.rb

@@ -0,0 +1,19 @@
+module Indexers
+  class ComputedSorts
+
+    def add(name, &block)
+      registry[name] = block
+    end
+
+    def find(name)
+      registry[name]
+    end
+
+    private
+
+    def registry
+      @registry ||= {}
+    end
+
+  end
+end

data/lib/indexers/concern.rb

@@ -0,0 +1,30 @@
+module Indexers
+  module Concern
+    extend ActiveSupport::Concern
+
+    included do
+      after_commit :index, on: :create
+      after_commit :reindex, on: :update
+      after_commit :unindex, on: :destroy
+    end
+
+    %i(index reindex unindex).each do |name|
+      define_method name do
+        self.class.indexer.send name, self
+      end
+    end
+
+    module ClassMethods
+
+      def search(*args)
+        options = args.extract_options!
+        Collection.new indexer, args, options
+      end
+
+      def indexer
+        @indexer ||= Indexers.definitions.find(name.parameterize('_').to_sym)
+      end
+
+    end
+  end
+end

data/lib/indexers/configuration.rb

@@ -0,0 +1,35 @@
+module Indexers
+  class Configuration
+
+    attr_accessor :hosts, :log, :trace
+
+    def mappings(&block)
+      if block_given?
+        @mappings = Dsl::Api.new(&block).to_h
+      else
+        @mappings
+      end
+    end
+
+    def analysis(&block)
+      if block_given?
+        @analysis = { analysis: Dsl::Api.new(&block).to_h }
+      else
+        @analysis
+      end
+    end
+
+    def suggestions(&block)
+      if block_given?
+        @suggestions = block
+      else
+        @suggestions
+      end
+    end
+
+    def computed_sort(*args, &block)
+      Indexers.computed_sorts.add *args, &block
+    end
+
+  end
+end