elasticsearch-model 0.1.1 → 0.1.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NTEzZGEzOThmZGEyNGQ1NWU1NmQ5M2MxZjY2MTY3Zjc4MjY4YWUzMQ==
+  data.tar.gz: !binary |-
+    MzViYTk0YjhmZjA3ODI1MTlhOWZhNDU4NzdjZTQyZDIxNDcxODYwZg==
+SHA512:
+  metadata.gz: !binary |-
+    ODRhOGMxYzBmMTZlYWU4YzJmNTRjZmRjZjQ5Y2VkY2I5M2E0ZTFhZWZjYTVl
+    ODcyMDUyMzkyOGE5MmI2ZDcwNWIwNTEwMTY4YmZlMzIyOWU2NDU2OGI2NmM4
+    MGFkODMxM2I1NmZmNmRmMDRiNzViOGIyZDhkYzU2MGMxNDQ2NzA=
+  data.tar.gz: !binary |-
+    NGQ2YTM3YWVkNzVkYWU5MmNkNjJmMjZjZTM2MGMxZGU0NGY3Y2U5MmRlZTU5
+    NWZjOGY2ZDdmODIyMzYzNDkzOTM4M2E4M2YwYTkwN2FmMjZlNmQ2MTIyODRk
+    YjM4ZTQ1NWNkN2VkY2UzNDU4NThjYTY0NmMyOWY5NDA4MmU5MjA=

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.1.2
+
+* Properly delegate existence methods like `result.foo?` to `result._source.foo`
+* Exception is raised when `type` is not passed to Mappings#new
+* Allow passing an ActiveRecord scope to the `import` method
+* Added, that `each_with_hit` and `map_with_hit` in `Elasticsearch::Model::Response::Records` call `to_a`
+* Added support for [`will_paginate`](https://github.com/mislav/will_paginate) pagination library
+* Added the ability to transform models during indexing
+* Added explicit `type` and `id` methods to Response::Result, aliasing `_type` and `_id`
+
 ## 0.1.1
 
 * Improved documentation and tests

data/README.md

@@ -194,6 +194,13 @@ response.any? { |r| r.title =~ /fox|dog/ }
 # => true
 ```
 
+To use `Array`'s methods (including any _ActiveSupport_ extensions), just call `to_a` on the object:
+
+```ruby
+response.to_a.last.title
+# "Fast black dogs"
+```
+
 #### Search results as database records
 
 Instead of returning documents from Elasticsearch, the `records` method will return a collection
@@ -245,9 +252,10 @@ response.records.each_with_hit { |record, hit| puts "* #{record.title}: #{hit._s
 #### Pagination
 
 You can implement pagination with the `from` and `size` search parameters. However, search results
-can be automatically paginated with the [`kaminari`](http://rubygems.org/gems/kaminari) gem.
+can be automatically paginated with the [`kaminari`](http://rubygems.org/gems/kaminari) or
+[`will_paginate`](https://github.com/mislav/will_paginate) gems.
 
-If Kaminari is loaded, use the familiar paging methods:
+If Kaminari or WillPaginate is loaded, use the familiar paging methods:
 
 ```ruby
 response.page(2).results
@@ -264,7 +272,7 @@ In a Rails controller, use the the `params[:page]` parameter to paginate through
 @articles.next_page
 # => 3
 ```
-To initialize and include the pagination support manually:
+To initialize and include the Kaminari pagination support manually:
 
 ```ruby
 Kaminari::Hooks.init

data/elasticsearch-model.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "oj"
   s.add_development_dependency "kaminari"
+  s.add_development_dependency "will_paginate"
   # NOTE: Do not add Mongoid here, keep only in 3/4 files
 
   s.add_development_dependency "minitest", "~> 4.0"

data/lib/elasticsearch/model.rb

@@ -31,8 +31,11 @@ require 'elasticsearch/model/response/pagination'
 
 require 'elasticsearch/model/ext/active_record'
 
-if defined?(::Kaminari)
+case
+when defined?(::Kaminari)
   Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::Kaminari
+when defined?(::WillPaginate)
+  Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::WillPaginate
 end
 
 module Elasticsearch

data/lib/elasticsearch/model/adapters/active_record.rb

@@ -83,15 +83,20 @@ module Elasticsearch
           # @see http://api.rubyonrails.org/classes/ActiveRecord/Batches.html ActiveRecord::Batches.find_in_batches
           #
           def __find_in_batches(options={}, &block)
-            find_in_batches(options) do |batch|
-              batch_for_bulk = batch.map { |a| { index: { _id: a.id, data: a.__elasticsearch__.as_indexed_json } } }
-              yield batch_for_bulk
+            named_scope = options.delete(:scope)
+
+            scope = named_scope ? self.__send__(named_scope) : self
+
+            scope.find_in_batches(options) do |batch|
+              yield batch
             end
           end
-        end
 
+          def __transform
+            lambda { |model|  { index: { _id: model.id, data: model.__elasticsearch__.as_indexed_json } } }
+          end
+        end
       end
-
     end
   end
 end

data/lib/elasticsearch/model/adapters/default.rb

@@ -36,6 +36,12 @@ module Elasticsearch
           def __find_in_batches(options={}, &block)
             raise NotImplemented, "Method not implemented for default adapter"
           end
+
+          # @abstract Implement this method in your adapter
+          #
+          def __transform
+            raise NotImplemented, "Method not implemented for default adapter"
+          end
         end
 
       end

data/lib/elasticsearch/model/adapters/mongoid.rb

@@ -70,17 +70,19 @@ module Elasticsearch
               items << item
 
               if items.length % options[:batch_size] == 0
-                batch_for_bulk = items.map { |a| { index: { _id: a.id.to_s, data: a.as_indexed_json } } }
-                yield batch_for_bulk
+                yield items
                 items = []
               end
             end
 
             unless items.empty?
-              batch_for_bulk = items.map { |a| { index: { _id: a.id.to_s, data: a.as_indexed_json } } }
-              yield batch_for_bulk
+              yield items
             end
           end
+
+          def __transform
+            lambda {|a|  { index: { _id: a.id.to_s, data: a.as_indexed_json } }}
+          end
         end
 
       end

data/lib/elasticsearch/model/importing.rb

@@ -64,11 +64,29 @@ module Elasticsearch
         #
         #    Article.import index: 'my-new-index', type: 'my-other-type'
         #
+        # @example Pass an ActiveRecord scope to limit the imported records
+        #
+        #    Article.import scope: 'published'
+        #
+        # @example Transform records during the import with a lambda
+        #
+        #    transform = lambda do |a|
+        #      {index: {_id: a.id, _parent: a.author_id, data: a.__elasticsearch__.as_indexed_json}}
+        #    end
+        #
+        #    Article.import transform: transform
+        #
         def import(options={}, &block)
           errors       = 0
-          refresh      = options.delete(:refresh) || false
-          target_index = options.delete(:index)   || index_name
-          target_type  = options.delete(:type)    || document_type
+          refresh      = options.delete(:refresh)   || false
+          target_index = options.delete(:index)     || index_name
+          target_type  = options.delete(:type)      || document_type
+          transform    = options.delete(:transform) || __transform
+
+          unless transform.respond_to?(:call)
+            raise ArgumentError,
+                  "Pass an object responding to `call` as the :transport option, #{transform.class} given"
+          end
 
           if options.delete(:force)
             self.create_index! force: true, index: target_index
@@ -78,7 +96,7 @@ module Elasticsearch
             response = client.bulk \
                          index:   target_index,
                          type:    target_type,
-                         body:    batch
+                         body:    __batch_to_bulk(batch, transform)
 
             yield response if block_given?
 
@@ -90,6 +108,9 @@ module Elasticsearch
           return errors
         end
 
+        def __batch_to_bulk(batch, transform)
+          batch.map { |model| transform.call(model) }
+        end
       end
 
     end

data/lib/elasticsearch/model/indexing.rb

@@ -37,6 +37,8 @@ module Elasticsearch
         attr_accessor :options
 
         def initialize(type, options={})
+          raise ArgumentError, "`type` is missing" if type.nil?
+
           @type    = type
           @options = options
           @mapping = {}

data/lib/elasticsearch/model/response/pagination.rb

@@ -93,6 +93,71 @@ module Elasticsearch
             results.total
           end
         end
+
+        # Allow models to be paginated with the "will_paginate" gem [https://github.com/mislav/will_paginate]
+        #
+        module WillPaginate
+          def self.included(base)
+            base.__send__ :include, ::WillPaginate::CollectionMethods
+
+            # Include the paging methods in results and records
+            #
+            methods = [:current_page, :per_page, :total_entries, :total_pages, :previous_page, :next_page, :out_of_bounds?]
+            Elasticsearch::Model::Response::Results.__send__ :delegate, *methods, to: :response
+            Elasticsearch::Model::Response::Records.__send__ :delegate, *methods, to: :response
+          end
+
+          # Main pagination method
+          #
+          # @example
+          #
+          #     Article.search('foo').paginate(page: 1, per_page: 30)
+          #
+          def paginate(options)
+            page = [options[:page].to_i, 1].max
+            per_page = (options[:per_page] || klass.per_page).to_i
+
+            search.definition.update size: per_page,
+                                     from: (page - 1) * per_page
+            self
+          end
+
+          # Return the current page
+          #
+          def current_page
+            search.definition[:from] / per_page + 1 if search.definition[:from] && per_page
+          end
+
+          # Pagination method
+          #
+          # @example
+          #
+          #     Article.search('foo').page(2)
+          #
+          def page(num)
+            paginate(page: num, per_page: per_page) # shorthand
+          end
+
+          # Return or set the "size" value
+          #
+          # @example
+          #
+          #     Article.search('foo').per_page(15).page(2)
+          #
+          def per_page(num = nil)
+            if num.nil?
+              search.definition[:size]
+            else
+              paginate(page: current_page, per_page: num) # shorthand
+            end
+          end
+
+          # Returns the total number of results
+          #
+          def total_entries
+            results.total
+          end
+        end
       end
 
     end

data/lib/elasticsearch/model/response/records.rb

@@ -43,13 +43,13 @@ module Elasticsearch
         # Yields [record, hit] pairs to the block
         #
         def each_with_hit(&block)
-          records.zip(results).each(&block)
+          records.to_a.zip(results).each(&block)
         end
 
         # Yields [record, hit] pairs and returns the result
         #
         def map_with_hit(&block)
-          records.zip(results).map(&block)
+          records.to_a.zip(results).map(&block)
         end
 
         # Delegate methods to `@records`

data/lib/elasticsearch/model/response/result.rb

@@ -17,14 +17,28 @@ module Elasticsearch
           @result = Hashie::Mash.new(attributes)
         end
 
+        # Return document `_id` as `id`
+        #
+        def id
+          @result['_id']
+        end
+
+        # Return document `_type` as `_type`
+        #
+        def type
+          @result['_type']
+        end
+
         # Delegate methods to `@result` or `@result._source`
         #
-        def method_missing(method_name, *arguments)
+        def method_missing(name, *arguments)
           case
-          when @result.respond_to?(method_name.to_sym)
-            @result.__send__ method_name.to_sym, *arguments
-          when @result._source && @result._source.respond_to?(method_name.to_sym)
-            @result._source.__send__ method_name.to_sym, *arguments
+          when name.to_s.end_with?('?')
+            @result.__send__(name, *arguments) || ( @result._source && @result._source.__send__(name, *arguments) )
+          when @result.respond_to?(name)
+            @result.__send__ name, *arguments
+          when @result._source && @result._source.respond_to?(name)
+            @result._source.__send__ name, *arguments
           else
             super
           end
@@ -43,7 +57,6 @@ module Elasticsearch
         end
 
         # TODO: #to_s, #inspect, with support for Pry
-
       end
     end
   end

data/lib/elasticsearch/model/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Model
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/test/integration/active_record_basic_test.rb

@@ -52,6 +52,19 @@ module Elasticsearch
           assert_equal 'Test', response.records.first.title
         end
 
+        should "provide access to result" do
+          response = Article.search query: { match: { title: 'test' } }, highlight: { fields: { title: {} } }
+
+          assert_equal 'Test', response.results.first.title
+
+          assert_equal true,  response.results.first.title?
+          assert_equal false, response.results.first.boo?
+
+          assert_equal true,  response.results.first.highlight?
+          assert_equal true,  response.results.first.highlight.title?
+          assert_equal false, response.results.first.highlight.boo?
+        end
+
         should "iterate over results" do
           response = Article.search('title:test')
 
@@ -59,6 +72,13 @@ module Elasticsearch
           assert_equal [1, 2],     response.records.map(&:id)
         end
 
+        should "return _id and _type as #id and #type" do
+          response = Article.search('title:test')
+
+          assert_equal '1',       response.results.first.id
+          assert_equal 'article', response.results.first.type
+        end
+
         should "access results from records" do
           response = Article.search('title:test')
 
@@ -68,6 +88,18 @@ module Elasticsearch
           end
         end
 
+        should "preserve the search results order for records" do
+          response = Article.search('title:code')
+
+          response.records.each_with_hit do |r, h|
+            assert_equal h._id, r.id.to_s
+          end
+
+          response.records.map_with_hit do |r, h|
+            assert_equal h._id, r.id.to_s
+          end
+        end
+
         should "remove document from index on destroy" do
           article = Article.first
 

data/test/integration/active_record_import_test.rb

@@ -7,9 +7,12 @@ module Elasticsearch
       class ::ImportArticle < ActiveRecord::Base
         include Elasticsearch::Model
 
+        scope :popular, -> { where('views >= 50') }
+
         mapping do
           indexes :title,      type: 'string'
           indexes :views,      type: 'integer'
+          indexes :numeric,    type: 'integer'
           indexes :created_at, type: 'date'
         end
       end
@@ -19,7 +22,8 @@ module Elasticsearch
           ActiveRecord::Schema.define(:version => 1) do
             create_table :import_articles do |t|
               t.string   :title
-              t.string   :views # For the sake of invalid data sent to Elasticsearch
+              t.integer  :views
+              t.string   :numeric # For the sake of invalid data sent to Elasticsearch
               t.datetime :created_at, :default => 'NOW()'
             end
           end
@@ -28,7 +32,7 @@ module Elasticsearch
           ImportArticle.__elasticsearch__.create_index! force: true
           ImportArticle.__elasticsearch__.client.cluster.health wait_for_status: 'yellow'
 
-          100.times { |i| ImportArticle.create! title: "Test #{i}" }
+          100.times { |i| ImportArticle.create! title: "Test #{i}", views: i }
         end
 
         should "import all the documents" do
@@ -49,8 +53,17 @@ module Elasticsearch
           assert_equal 100, ImportArticle.search('*').results.total
         end
 
+        should "import only documents from a specific scope" do
+          assert_equal 100, ImportArticle.count
+
+          assert_equal 0, ImportArticle.import(scope: 'popular')
+
+          ImportArticle.__elasticsearch__.refresh_index!
+          assert_equal 50, ImportArticle.search('*').results.total
+        end
+
         should "report and not store/index invalid documents" do
-          ImportArticle.create! title: "Test INVALID", views: "INVALID"
+          ImportArticle.create! title: "Test INVALID", numeric: "INVALID"
 
           assert_equal 101, ImportArticle.count
 
@@ -68,6 +81,18 @@ module Elasticsearch
           ImportArticle.__elasticsearch__.refresh_index!
           assert_equal 100, ImportArticle.search('*').results.total
         end
+
+        should "transform documents with the option" do
+          assert_equal 100, ImportArticle.count
+
+          assert_equal 0, ImportArticle.import( transform: ->(a) {{ index: { data: { name: a.title, foo: 'BAR' } }}} )
+
+          ImportArticle.__elasticsearch__.refresh_index!
+          assert_contains ImportArticle.search('*').results.first._source.keys, 'name'
+          assert_contains ImportArticle.search('*').results.first._source.keys, 'foo'
+          assert_equal 100, ImportArticle.search('test').results.total
+          assert_equal 100, ImportArticle.search('bar').results.total
+        end
       end
 
     end