elastic_record 0.2.3 → 0.3.0

data/lib/elastic_record.rb

@@ -8,3 +8,5 @@ module ElasticRecord
   autoload :Relation, 'elastic_record/relation'
   autoload :Searching, 'elastic_record/searching'
 end
+
+require 'elastic_record/railtie' if defined?(Rails)

data/lib/elastic_record/connection.rb

@@ -1,10 +1,9 @@
 module ElasticRecord
   module Connection
     def elastic_connection
-      ElasticRecord::Config.connection_options
       @elastic_connection ||= ElasticSearch.new(
         ElasticRecord::Config.servers,
-        ElasticRecord::Config.connection_options.merge(index: model_name.collection, type: model_name.element)
+        ElasticRecord::Config.connection_options.merge(index: elastic_index.alias_name, type: elastic_index.type)
       )
     end
 

data/lib/elastic_record/index.rb

@@ -1,8 +1,10 @@
+require 'elastic_record/index/documents'
 require 'elastic_record/index/manage'
 require 'elastic_record/index/mapping'
 
 module ElasticRecord
   class Index
+    include Documents
     include Manage
     include Mapping
 
@@ -12,12 +14,42 @@ module ElasticRecord
       @model = model
     end
 
-    def name
+    def alias_name
       model.model_name.collection
     end
 
     def type
       model.model_name.element
     end
+
+    private
+      def new_index_name
+        "#{alias_name}_#{Time.now.to_i}"
+      end
+
+      def json_get(path)
+        ActiveSupport::JSON.decode http.get(path).body
+      end
+
+      def json_post(path, json)
+        http.post(path, ActiveSupport::JSON.encode(json))
+      end
+
+      def json_put(path, json)
+        http.put(path, ActiveSupport::JSON.encode(json))
+      end
+
+      def json_delete(path)
+        ActiveSupport::JSON.decode http.delete(path).body
+      end
+
+      def connection
+        @model.elastic_connection
+      end
+
+      def http
+        host, port = connection.current_server.split ':'
+        Net::HTTP.new(host, port)
+      end
   end
 end

data/lib/elastic_record/index/documents.rb

@@ -0,0 +1,34 @@
+module ElasticRecord
+  class Index
+    module Documents
+      def index_record(record, index_name = nil)
+        index_name ||= alias_name
+        document = record.respond_to?(:as_search) ? record.as_search : record.default_as_search
+
+        connection.index(document, id: record.id, index: index_name)
+        # json_put "#{index_name}/#{type}/#{record.id}", document
+      end
+      
+      def delete_record(record, index_name = nil)
+        index_name ||= alias_name
+
+        connection.delete(record.id, index: index_name)
+        # json_delete "#{index_name}/#{type}/#{record.id}"
+      end
+
+      def record_exists?(id)
+        !connection.get(id).nil?
+      end
+
+      def bulk_add(batch, index_name = nil)
+        index_name ||= alias_name
+
+        connection.bulk do
+          batch.each do |record|
+            index_record(record, index_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_record/index/manage.rb

@@ -1,21 +1,69 @@
 module ElasticRecord
   class Index
     module Manage
-      def create(index_name = name)
-        model.elastic_connection.create_index(index_name)
+      def create_and_deploy(index_name = new_index_name)
+        create(index_name)
+        deploy(index_name)
+        index_name
+      end
+
+      def create(index_name = new_index_name)
+        http.put("/#{index_name}", '')
+        update_mapping(index_name)
+        index_name
+      end
+
+      def delete(index_name)
+        http.delete("/#{index_name}")
+      end
 
-        unless mapping.empty?
-          model.elastic_connection.update_mapping(mapping, index: index_name)
+      def delete_all
+        all_names.each do |index_name|
+          delete index_name
         end
       end
 
-      def delete(index_name = name)
-        model.elastic_connection.delete_index(index_name)
+      def exists?(index_name)
+        http.head("/#{index_name}").code == '200'
       end
 
-      def alias
-        # alias_actions = {add: {index_name => pending_index_alias}}
-        # elastic_connection.alias_index(alias_actions)
+      def deploy(index_name)
+        actions = [
+          {
+            add: {
+              "index" => index_name,
+              "alias" => alias_name
+            }
+          }
+        ]
+
+        aliased_indexes.each do |index_to_remove|
+          actions << {
+            remove: {
+              "index" => index_to_remove,
+              "alias" => alias_name
+            }
+          }
+        end
+
+        json_post '/_aliases', actions: actions
+      end
+
+      def update_mapping(index_name)
+        connection.update_mapping(mapping, index: index_name)
+        # json_put "/#{index_name}/#{type}/_mapping", type => mapping
+      end
+
+      def aliased_indexes
+        json = json_get '/_cluster/state'
+        json["metadata"]["indices"].select { |name, status| status["aliases"].include?(alias_name) }.map { |name, status| name }
+      end
+
+      def all_names
+        json = json_get '/_status'
+
+        regex = %r{^#{alias_name}_?}
+        json['indices'].keys.grep(regex)
       end
     end
   end

data/lib/elastic_record/railtie.rb

@@ -0,0 +1,5 @@
+module ElasticRecord
+  class Railtie < Rails::Railtie
+    rake_tasks { load "elastic_record/tasks/index.rake" }
+  end
+end

data/lib/elastic_record/relation/batches.rb

@@ -1,13 +1,26 @@
 module ElasticRecord
   module Batches
     def find_each
-      options = {scroll: '20m', size: 100, search_type: 'scan'}
+      find_in_batches do |records|
+        records.each { |record| yield record }
+      end
+    end
+
+    def find_in_batches(options = {})
+      options[:size] ||= 100
+      options[:scroll]  ||= '20m'
+      options[:search_type] = 'scan'
 
       hits = klass.elastic_connection.search(as_elastic, options)
 
       klass.elastic_connection.scroll(hits.scroll_id, scroll: options[:scroll], ids_only: true) do |hits|
-        records = klass.find(hits.to_a)
-        records.each { |record| yield record }
+        yield klass.find(hits.to_a)
+      end
+    end
+
+    def reindex
+      relation.find_in_batches do |batch|
+        elastic_index.bulk_add(batch)
       end
     end
   end

data/test/elastic_record/connection_test.rb

@@ -5,6 +5,6 @@ class ElasticRecord::ConnectionTest < MiniTest::Spec
     connection = Widget.elastic_connection
 
     assert_equal Widget.elastic_index.type, connection.default_type
-    assert_equal Widget.elastic_index.name, connection.default_index
+    assert_equal Widget.elastic_index.alias_name, connection.default_index
   end
 end

data/test/elastic_record/index/documents_test.rb

@@ -0,0 +1,42 @@
+require 'helper'
+
+class ElasticRecord::Index::DocumentsTest < MiniTest::Spec
+  def setup
+    super
+    
+    Widget.reset_index!
+  end
+
+  def test_index_record
+    record = Widget.new(id: 'abc', color: 'red')
+
+    index.index_record(record)
+
+    assert index.record_exists?('abc')
+    refute index.record_exists?('xyz')
+  end
+
+  def test_delete_record
+    record = Widget.new(id: 'abc', color: 'red')
+
+    index.index_record(record)
+    assert index.record_exists?('abc')
+
+    index.delete_record(record)
+    refute index.record_exists?('abc')
+  end
+
+  def test_bulk
+    record = Widget.new(id: 'abc', color: 'red')
+    
+    index.bulk_add [record]
+
+    assert index.record_exists?('abc')
+    refute index.record_exists?('xyz')
+  end
+
+  private
+    def index
+      @index ||= Widget.elastic_index
+    end
+end

data/test/elastic_record/index/manage_test.rb

@@ -1,4 +1,38 @@
 require 'helper'
 
 class ElasticRecord::Index::ManageTest < MiniTest::Spec
+  def setup
+    super
+
+    index.delete_all
+  end
+  
+  def test_create
+    assert !index.exists?('widgets_foo')
+
+    index.create 'widgets_foo'
+
+    assert index.exists?('widgets_foo')
+  end
+
+  def test_exists
+    index.create 'widgets_foo'
+
+    assert index.exists?('widgets_foo')
+    assert !index.exists?('widgets_bar')
+  end
+
+  def test_deploy
+    index.create 'widgets_foo'
+    
+    assert index.aliased_indexes.empty?
+    index.deploy 'widgets_foo'
+
+    assert_equal ['widgets_foo'], index.aliased_indexes
+  end
+
+  private
+    def index
+      @index ||= Widget.elastic_index
+    end
 end

data/test/elastic_record/index_test.rb

@@ -1,10 +1,10 @@
 require 'helper'
 
 class ElasticRecord::IndexTest < MiniTest::Spec
-  def test_elastic_connection
+  def test_model_name
     index = Widget.elastic_index
 
+    assert_equal 'widgets', index.alias_name
     assert_equal 'widget', index.type
-    assert_equal 'widgets', index.name
   end
 end

data/test/elastic_record/relation/batches_test.rb

@@ -14,19 +14,29 @@ class ElasticRecord::Relation::BatchesTest < MiniTest::Spec
     assert_equal ['5', '10', '15'].to_set, results.to_set
   end
 
-  def test_find_each_with_scope
+  def test_find_in_batches
     results = []
-    Widget.elastic_relation.filter(color: %w(red blue)).find_each do |widget|
-      results << widget.id
+    Widget.elastic_relation.find_in_batches do |widgets|
+      results << widgets.map(&:id)
+    end
+    assert_equal [['5', '10', '15'].to_set], results.map(&:to_set)
+  end
+
+  def test_find_in_batches_with_scope
+    results = []
+    Widget.elastic_relation.filter(color: %w(red blue)).find_in_batches do |widgets|
+      results << widgets.map(&:id)
     end
-    assert_equal ['5', '10'].to_set, results.to_set
+    assert_equal [['5', '10'].to_set], results.map(&:to_set)
   end
 
   private
     def create_widgets
-      Widget.elastic_connection.index({'widget' => {'color' => 'red'}}, {index: 'widgets', type: 'widget', id: 5})
-      Widget.elastic_connection.index({'widget' => {'color' => 'blue'}}, {index: 'widgets', type: 'widget', id: 10})
-      Widget.elastic_connection.index({'widget' => {'color' => 'green'}}, {index: 'widgets', type: 'widget', id: 15})
+      Widget.elastic_index.bulk_add [
+        Widget.new(id: 5, color: 'red'),
+        Widget.new(id: 10, color: 'blue'),
+        Widget.new(id: 15, color: 'green'),
+      ]
       
       Widget.elastic_connection.refresh
     end

data/test/elastic_record/relation/search_methods_test.rb

@@ -162,10 +162,10 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Spec
     Widget.elastic_connection.index({'widget' => {'color' => 'red'}}, {index: 'widgets', type: 'widget', id: 5})
     Widget.elastic_connection.index({'widget' => {'color' => 'blue'}}, {index: 'widgets', type: 'widget', id: 10})
 
-    records = relation.select { |record| record.id == '5' }
+    records = relation.select { |record| record.id == '10' }
 
     assert_equal 1, records.size
-    assert_equal '5', records.first.id
+    assert_equal '10', records.first.id
   end
 
   def test_extending_with_block

data/test/support/widget.rb

@@ -2,6 +2,11 @@ class Widget
   extend ActiveModel::Naming
   include ElasticRecord::Model
 
+  self.elastic_index.mapping[:properties].update(color: {
+    type: 'string',
+    index: 'not_analyzed'
+  })
+
   class << self
     def find(ids)
       ids.map { |id| new(id: id, color: 'red') }
@@ -18,24 +23,8 @@ class Widget
     end
 
     def reset_index!
-      elastic_index.delete
-      elastic_index.create
-      elastic_connection.update_mapping(
-        {
-          properties: {
-            color: {
-              type: 'string',
-              index: 'not_analyzed'
-            }
-          },
-          _source: {
-            enabled: false
-          }
-        },  
-        {
-          index: 'widgets'
-        }
-      )
+      elastic_index.delete_all
+      elastic_index.create_and_deploy
     end
   end
 
@@ -45,4 +34,8 @@ class Widget
       send("#{key}=", val)
     end
   end
+
+  def as_search
+    {color: color}
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_record
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-14 00:00:00.000000000 Z
+date: 2012-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: arelastic
@@ -68,11 +68,13 @@ extra_rdoc_files:
 files:
 - lib/elastic_record/config.rb
 - lib/elastic_record/connection.rb
+- lib/elastic_record/index/documents.rb
 - lib/elastic_record/index/manage.rb
 - lib/elastic_record/index/mapping.rb
 - lib/elastic_record/index.rb
 - lib/elastic_record/model.rb
 - lib/elastic_record/orm/active_record.rb
+- lib/elastic_record/railtie.rb
 - lib/elastic_record/relation/batches.rb
 - lib/elastic_record/relation/delegation.rb
 - lib/elastic_record/relation/finder_methods.rb
@@ -84,6 +86,7 @@ files:
 - lib/elastic_record.rb
 - test/elastic_record/config_test.rb
 - test/elastic_record/connection_test.rb
+- test/elastic_record/index/documents_test.rb
 - test/elastic_record/index/manage_test.rb
 - test/elastic_record/index/mapping_test.rb
 - test/elastic_record/index_test.rb