elastic_record 0.5.0 → 0.6.0

data/.travis.yml

@@ -0,0 +1,4 @@
+services:
+  - elasticsearch
+rvm:
+  - 1.9.3

data/Gemfile

@@ -1,4 +1,3 @@
 source :rubygems
-
 gemspec
-# gem 'arelastic', path: '~/code/arelastic'
+gem 'rake'

data/README.rdoc

@@ -1,4 +1,5 @@
 = ElasticRecord
+{<img src="https://secure.travis-ci.org/matthuhiggins/elastic_record.png?rvm=1.9.3" />}[http://travis-ci.org/matthuhiggins/elastic_record]
 
 ElasticRecord is an elasticsearch ORM.
 

data/elastic_record.gemspec

@@ -2,7 +2,7 @@
  
 Gem::Specification.new do |s|
   s.name = 'elastic_record'
-  s.version = '0.5.0'
+  s.version = '0.6.0'
   s.summary = 'Use Elastic Search with your objects'
   s.description = 'Find your records with elastic search'
 
@@ -19,6 +19,5 @@ Gem::Specification.new do |s|
   s.test_files  = `git ls-files -- {test}/*`.split("\n")
 
   s.add_dependency 'arelastic'
-  s.add_dependency 'rubberband', '>= 0.1.1'
   s.add_dependency 'activemodel'
 end

data/lib/elastic_record/callbacks.rb

@@ -3,11 +3,11 @@ module ElasticRecord
     def self.included(base)
       base.class_eval do
         after_save do
-          self.class.elastic_index.index_record self
+          self.class.elastic_index.index_document id, as_search
         end
 
         after_destroy do
-          self.class.elastic_index.delete_record self
+          self.class.elastic_index.delete_document id
         end
       end
     end

data/lib/elastic_record/connection.rb

@@ -1,14 +1,70 @@
 module ElasticRecord
-  module Connection
-    def elastic_connection
-      @elastic_connection ||= ElasticSearch.new(
-        ElasticRecord::Config.servers,
-        ElasticRecord::Config.connection_options.merge(index: elastic_index.alias_name, type: elastic_index.type)
-      )
+  class Connection
+    # :timeout: 10
+    # :retries: 2
+    # :auto_discovery: false
+
+    attr_accessor :servers, :options
+    def initialize(servers, options = {})
+      if servers.is_a?(Array)
+        self.servers = servers
+      else
+        self.servers = servers.split(',')
+      end
+
+      self.options = options
+    end
+
+    def head(path)
+      http_request(Net::HTTP::Head, path).code
+    end
+
+    def json_get(path, json = nil)
+      json_request Net::HTTP::Get, path, json
+    end
+
+    def json_post(path, json = nil)
+      json_request Net::HTTP::Post, path, json
+    end
+
+    def json_put(path, json = nil)
+      json_request Net::HTTP::Put, path, json
     end
 
-    def elastic_connection=(connection)
-      @elastic_connection = connection
+    def json_delete(path, json = nil)
+      json_request Net::HTTP::Delete, path, json
     end
+
+    def json_request(request_klass, path, json)
+      body = json.is_a?(Hash) ? ActiveSupport::JSON.encode(json) : json
+      response = http_request(request_klass, path, body)
+      json = ActiveSupport::JSON.decode response.body
+
+      raise json['error'] if json['error']
+
+      json
+    end
+
+    def http_request(request_klass, path, body = nil)
+      request = request_klass.new(path)
+      request.body = body
+
+      http.request(request)
+    end
+
+    private
+      def choose_server
+        servers.sample
+      end
+
+      def http
+        host, port = choose_server.split ':'
+
+        http = Net::HTTP.new(host, port)
+        if options[:timeout]
+          http.read_timeout = options[:timeout].to_i
+        end
+        http
+      end
   end
 end

data/lib/elastic_record/index/documents.rb

@@ -1,25 +1,47 @@
+require 'active_support/core_ext/object/to_query'
+
 module ElasticRecord
   class Index
     module Documents
-      def index_record(record, index_name = nil)
+      def index_document(id, document, index_name = nil)
         return if disabled
 
         index_name ||= alias_name
-        document = record.respond_to?(:as_search) ? record.as_search : {}
 
-        connection.index(document, id: record.id, index: index_name)
-        # json_put "#{index_name}/#{type}/#{record.id}", document
+        if @batch
+          @batch << { index: { _index: index_name, _type: type, _id: id } }
+          @batch << document
+        else
+          connection.json_put "/#{index_name}/#{type}/#{id}", document
+        end
       end
       
-      def delete_record(record, index_name = nil)
+      def delete_document(id, index_name = nil)
         index_name ||= alias_name
 
-        connection.delete(record.id, index: index_name)
-        # json_delete "#{index_name}/#{type}/#{record.id}"
+        if @batch
+          @batch << { delete: { _index: index_name, _type: type, _id: id } }
+        else
+          connection.json_delete "/#{index_name}/#{type}/#{id}"
+        end
       end
 
       def record_exists?(id)
-        !connection.get(id).nil?
+        connection.json_get("/#{alias_name}/#{type}/#{id}")['exists']
+      end
+
+      def search(elastic_query, options = {})
+        url = "/#{alias_name}/#{type}/_search"
+        if options.any?
+          url += "?#{options.to_query}"
+        end
+
+        connection.json_get url, elastic_query
+      end
+
+      def scroll(scroll_id, scroll_keep_alive)
+        options = {scroll_id: scroll_id, scroll: scroll_keep_alive}
+        connection.json_get("/_search/scroll?#{options.to_query}")
       end
 
       def bulk_add(batch, index_name = nil)
@@ -27,12 +49,23 @@ module ElasticRecord
 
         index_name ||= alias_name
 
-        connection.bulk do
+        bulk do
           batch.each do |record|
-            index_record(record, index_name)
+            index_document(record.id, record.as_search, index_name)
           end
         end
       end
+
+      def bulk
+        @batch = []
+        yield
+        if @batch.any?
+          body = @batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
+          connection.json_post "/_bulk", body
+        end
+      ensure
+        @batch = nil
+      end
     end
   end
 end

data/lib/elastic_record/index/manage.rb

@@ -8,13 +8,13 @@ module ElasticRecord
       end
 
       def create(index_name = new_index_name)
-        json_put "/#{index_name}"
+        connection.json_put "/#{index_name}"
         update_mapping(index_name)
         index_name
       end
 
       def delete(index_name)
-        json_delete "/#{index_name}"
+        connection.json_delete "/#{index_name}"
       end
 
       def delete_all
@@ -24,7 +24,7 @@ module ElasticRecord
       end
 
       def exists?(index_name)
-        head("/#{index_name}") == '200'
+        connection.head("/#{index_name}") == '200'
       end
 
       def deploy(index_name)
@@ -46,15 +46,15 @@ module ElasticRecord
           }
         end
 
-        json_post '/_aliases', actions: actions
+        connection.json_post '/_aliases', actions: actions
       end
 
       def update_mapping(index_name)
-        json_put "/#{index_name}/#{type}/_mapping", type => mapping
+        connection.json_put "/#{index_name}/#{type}/_mapping", type => mapping
       end
 
       def refresh
-        connection.refresh
+        connection.json_post "/#{alias_name}/_refresh"
       end
 
       def reset
@@ -63,12 +63,12 @@ module ElasticRecord
       end
 
       def aliased_names
-        json = json_get '/_cluster/state'
+        json = connection.json_get '/_cluster/state'
         json["metadata"]["indices"].select { |name, status| status["aliases"].include?(alias_name) }.map { |name, status| name }
       end
 
       def all_names
-        json = json_get '/_status'
+        json = connection.json_get '/_status'
 
         regex = %r{^#{alias_name}_?}
         json['indices'].keys.grep(regex)

data/lib/elastic_record/index/percolator.rb

@@ -8,19 +8,19 @@ module ElasticRecord
           update_mapping percolator_index_name
         end
 
-        json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
+        connection.json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
       end
 
       def delete_percolator(name)
-        json_delete "/_percolator/#{percolator_index_name}/#{name}"
+        connection.json_delete "/_percolator/#{percolator_index_name}/#{name}"
       end
 
       def percolator_exists?(name)
-        json_get("/_percolator/#{percolator_index_name}/#{name}")['exists']
+        connection.json_get("/_percolator/#{percolator_index_name}/#{name}")['exists']
       end
 
       def percolate(document)
-        json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
+        connection.json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
       end
 
       def percolator_index_name

data/lib/elastic_record/index.rb

@@ -35,55 +35,13 @@ module ElasticRecord
       @disabled = false
     end
 
-    # private
+    private
       def new_index_name
         "#{alias_name}_#{Time.now.to_i}"
       end
 
-      def head(path)
-        http_request(Net::HTTP::Head, path).code
-      end
-
-      def json_get(path, json = nil)
-        json_request Net::HTTP::Get, path, json
-      end
-
-      def json_post(path, json = nil)
-        json_request Net::HTTP::Post, path, json
-      end
-
-      def json_put(path, json = nil)
-        json_request Net::HTTP::Put, path, json
-      end
-
-      def json_delete(path, json = nil)
-        json_request Net::HTTP::Delete, path, json
-      end
-
-      def json_request(request_klass, path, json)
-        body = json ? ActiveSupport::JSON.encode(json) : nil
-        response = http_request(request_klass, path, body)
-        json = ActiveSupport::JSON.decode response.body
-
-        raise json['error'] if json['error']
-
-        json
-      end
-
-      def http_request(request_klass, path, body = nil)
-        request = request_klass.new(path)
-        request.body = body
-
-        http.request(request)
-      end
-
       def connection
-        @model.elastic_connection
-      end
-
-      def http
-        host, port = connection.current_server.split ':'
-        Net::HTTP.new(host, port)
+        model.elastic_connection
       end
   end
 end

data/lib/elastic_record/model.rb

@@ -2,11 +2,19 @@ module ElasticRecord
   module Model
     def self.included(base)
       base.class_eval do
-        extend Connection, Searching, ClassMethods
+        extend Searching, ClassMethods
       end
     end
 
     module ClassMethods
+      def elastic_connection
+        @elastic_connection ||= ElasticRecord::Connection.new(ElasticRecord::Config.servers)
+      end
+
+      def elastic_connection=(connection)
+        @elastic_connection = connection
+      end
+
       def elastic_relation
         ElasticRecord::Relation.new(self, arelastic)
       end

data/lib/elastic_record/relation/batches.rb

@@ -8,14 +8,18 @@ module ElasticRecord
       end
 
       def find_in_batches(options = {})
-        options[:size] ||= 100
-        options[:scroll]  ||= '20m'
-        options[:search_type] = 'scan'
+        scroll_keep_alive = '10m'
 
-        hits = klass.elastic_connection.search(as_elastic, options)
+        options = {
+          scroll: scroll_keep_alive,
+          size: 100,
+          search_type: 'scan'
+        }
 
-        klass.elastic_connection.scroll(hits.scroll_id, scroll: options[:scroll], ids_only: true) do |hits|
-          yield klass.find(hits.to_a)
+        scroll_id = klass.elastic_index.search(as_elastic, options)['_scroll_id']
+
+        while (hit_ids = get_scroll_hit_ids(scroll_id, scroll_keep_alive)).any?
+          yield klass.find(hit_ids)
         end
       end
 
@@ -24,6 +28,12 @@ module ElasticRecord
           elastic_index.bulk_add(batch)
         end
       end
+
+      private
+        def get_scroll_hit_ids(scroll_id, scroll_keep_alive)
+          json = klass.elastic_index.scroll(scroll_id, scroll_keep_alive)
+          json['hits']['hits'].map { |hit| hit['_id'] }
+        end
     end
   end
-end
+end

data/lib/elastic_record/relation.rb

@@ -18,11 +18,11 @@ module ElasticRecord
     end
 
     def count
-      to_hits.total_entries
+      search_results['hits']['total']
     end
 
     def facets
-      to_hits.facets
+      search_results['facets']
     end
 
     def create_percolator(name)
@@ -30,7 +30,7 @@ module ElasticRecord
     end
 
     def reset
-      @hits = @records = nil
+      @search_results = @records = nil
     end
 
     def initialize_copy(other)
@@ -46,11 +46,7 @@ module ElasticRecord
     end
 
     def to_ids
-      to_hits.to_a.map(&:id)
-    end
-
-    def to_hits
-      @hits ||= klass.elastic_connection.search(as_elastic)#, ids_only: true)
+      search_hits.map { |hit| hit['_id'] }
     end
 
     def ==(other)
@@ -72,5 +68,14 @@ module ElasticRecord
     ensure
       klass.current_elastic_search = previous
     end
+
+    private
+      def search_hits
+        search_results['hits']['hits']
+      end
+
+      def search_results
+        @search_results ||= klass.elastic_index.search(as_elastic)
+      end
   end
 end

data/lib/elastic_record.rb

@@ -1,5 +1,4 @@
 require 'arelastic'
-require 'rubberband'
 require 'active_support/core_ext/object/blank' # required because ActiveModel depends on this but does not require it
 require 'active_model'
 

data/test/elastic_record/callbacks_test.rb

@@ -17,7 +17,7 @@ class ElasticRecord::CallbacksTest < MiniTest::Spec
 
   def test_deleted_from_index
     widget = Widget.new id: '10', color: 'green'
-    Widget.elastic_index.index_record(widget)
+    Widget.elastic_index.index_document(widget.id, widget.as_search)
 
     assert Widget.elastic_index.record_exists?(widget.id)
 

data/test/elastic_record/connection_test.rb

@@ -1,10 +1,8 @@
 require 'helper'
 
 class ElasticRecord::ConnectionTest < MiniTest::Spec
-  def test_elastic_connection
-    connection = Widget.elastic_connection
-
-    assert_equal Widget.elastic_index.type, connection.default_type
-    assert_equal Widget.elastic_index.alias_name, connection.default_index
+  def test_servers
+    assert_equal ['foo', 'bar'], ElasticRecord::Connection.new('foo,bar').servers
+    assert_equal ['foo', 'bar'], ElasticRecord::Connection.new(['foo', 'bar']).servers
   end
 end

data/test/elastic_record/index/documents_test.rb

@@ -7,22 +7,18 @@ class ElasticRecord::Index::DocumentsTest < MiniTest::Spec
     index.reset
   end
 
-  def test_index_record
-    record = Widget.new(id: 'abc', color: 'red')
-
-    index.index_record(record)
+  def test_index_document
+    index.index_document('abc', color: 'red')
 
     assert index.record_exists?('abc')
     refute index.record_exists?('xyz')
   end
 
-  def test_delete_record
-    record = Widget.new(id: 'abc', color: 'red')
-
-    index.index_record(record)
+  def test_delete_document
+    index.index_document('abc', color: 'red')
     assert index.record_exists?('abc')
 
-    index.delete_record(record)
+    index.delete_document('abc')
     refute index.record_exists?('abc')
   end
 
@@ -35,6 +31,24 @@ class ElasticRecord::Index::DocumentsTest < MiniTest::Spec
     refute index.record_exists?('xyz')
   end
 
+  def test_bulk
+    assert_nil index.instance_variable_get(:@batch)
+
+    index.bulk do
+      index.index_document '5', color: 'green'
+      index.delete_document '3'
+
+      expected = [
+        {index: {_index: "widgets", _type: "widget", _id: "5"}},
+        {color: "green"},
+        {delete: {_index: "widgets", _type: "widget", _id: "3"}}
+      ]
+      assert_equal expected, index.instance_variable_get(:@batch)
+    end
+
+    assert_nil index.instance_variable_get(:@batch)
+  end
+
   private
     def index
       @index ||= Widget.elastic_index

data/test/elastic_record/model_test.rb

@@ -1,6 +1,13 @@
 require 'helper'
 
 class ElasticRecord::ModelTest < MiniTest::Spec
+  # def test_elastic_connection
+  #   connection = Widget.elastic_connection
+  # 
+  #   assert_equal Widget.elastic_index.type, connection.default_type
+  #   assert_equal Widget.elastic_index.alias_name, connection.default_index
+  # end
+
   def test_elastic_relation
     relation = Widget.elastic_relation
 

data/test/elastic_record/relation/delegation_test.rb

@@ -6,7 +6,7 @@ class ElasticRecord::Relation::DelegationTest < MiniTest::Spec
   end
 
   def test_delegate_to_array
-    Widget.elastic_index.index_record(Widget.new(id: 5, color: 'red'))
+    Widget.elastic_index.index_document('5', color: 'red')
     Widget.elastic_index.refresh
     
     records = []

data/test/elastic_record/relation/search_methods_test.rb

@@ -159,8 +159,10 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Spec
   end
 
   def test_select_with_block
-    Widget.elastic_connection.index({'widget' => {'color' => 'red'}}, {index: 'widgets', type: 'widget', id: 5})
-    Widget.elastic_connection.index({'widget' => {'color' => 'blue'}}, {index: 'widgets', type: 'widget', id: 10})
+    Widget.elastic_index.bulk_add [
+      Widget.new(id: 5, color: 'red'),
+      Widget.new(id: 10, color: 'blue')
+    ]
 
     records = relation.select { |record| record.id == '10' }
 

data/test/elastic_record/relation_test.rb

@@ -28,7 +28,7 @@ class ElasticRecord::RelationTest < MiniTest::Spec
   end
 
   def test_to_hits
-    assert Widget.elastic_relation.to_hits.is_a?(ElasticSearch::Api::Hits)
+    # assert Widget.elastic_relation.search_results.is_a?(ElasticSearch::Api::Hits)
   end
 
   def test_to_ids

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_record
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-07 00:00:00.000000000 Z
+date: 2012-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: arelastic
@@ -27,22 +27,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubberband
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.1.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +51,7 @@ extra_rdoc_files:
 - README.rdoc
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - MIT-LICENSE
 - README.rdoc