es-elasticity 0.2.11 → 0.3.0

data/lib/elasticity/strategies/single_index.rb

@@ -0,0 +1,97 @@
+module Elasticity
+  module Strategies
+    class SingleIndex
+      STATUSES = [:missing, :ok]
+
+      def initialize(client, index_name)
+        @client     = client
+        @index_name = index_name
+      end
+
+      def remap!
+        raise NotImplementedError
+      end
+
+      def missing?
+        not @client.index_exists(index: @index_name)
+      end
+
+      def create(index_def)
+        if missing?
+          @client.index_create(index: @index_name, body: index_def)
+        else
+          raise IndexError.new(@index_name, "index already exist")
+        end
+      end
+
+      def create_if_undefined(index_def)
+        create(index_def) if missing?
+      end
+
+      def delete
+        @client.index_delete(index: @index_name)
+      end
+
+      def delete_if_defined
+        delete unless missing?
+      end
+
+      def recreate(index_def)
+        delete_if_defined
+        create(index_def)
+      end
+
+      def index_document(type, id, attributes)
+        res = @client.index(index: @index_name, type: type, id: id, body: attributes)
+
+        if id = res["_id"]
+          [id, res["created"]]
+        else
+          raise IndexError.new(@update_alias, "failed to index document")
+        end
+      end
+
+      def delete_document(type, id)
+        @client.delete(index: @index_name, type: type, id: id)
+      end
+
+      def get_document(type, id)
+        @client.get(index: @index_name, type: type, id: id)
+      end
+
+      def search(type, body)
+        Search::Facade.new(@client, Search::Definition.new(@index_name, type, body))
+      end
+
+      def delete_by_query(type, body)
+        @client.delete_by_query(index: @index_name, type: type, body: body)
+      end
+
+      def bulk
+        b = Bulk::Index.new(@client, @index_name)
+        yield b
+        b.execute
+      end
+
+      def settings
+        args = { index: @index_name }
+        settings = @client.index_get_settings(index: @index_name)
+        settings[@index_name]["settings"]
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+
+      def mappings
+        args = { index: @index_name }
+        mapping = @client.index_get_mapping(index: @index_name)
+        mapping[@index_name]["mappings"]
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+
+      def flush
+        @client.index_flush(index: @index_name)
+      end
+    end
+  end
+end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "0.2.11"
+  VERSION = "0.3.0"
 end

data/spec/functional/persistence_spec.rb

@@ -0,0 +1,167 @@
+RSpec.describe "Persistence", elasticsearch: true do
+  describe "single index strategy" do
+    subject do
+      Class.new(Elasticity::Document) do
+        configure do |c|
+          c.index_base_name = "users"
+          c.document_type   = "user"
+          c.strategy        = Elasticity::Strategies::SingleIndex
+
+          c.mapping = {
+            properties: {
+              name: { type: "string" },
+              birthdate: { type: "date" },
+            },
+          }
+        end
+
+        attr_accessor :name, :birthdate
+
+        def to_document
+          { name: name, birthdate: birthdate }
+        end
+      end
+    end
+
+    before do
+      subject.recreate_index
+    end
+
+    after do
+      subject.delete_index
+    end
+
+    it "successfully index, update, search and delete" do
+      john = subject.new(name: "John", birthdate: "1985-10-31")
+      mari = subject.new(name: "Mari", birthdate: "1986-09-24")
+
+      john.update
+      mari.update
+
+      subject.flush_index
+
+      results = subject.search(sort: :name)
+      expect(results.total).to eq 2
+
+      expect(results[0]).to eq(john)
+      expect(results[1]).to eq(mari)
+
+      john.update
+      mari.delete
+
+      subject.flush_index
+
+      results = subject.search(sort: :name)
+      expect(results.total).to eq 1
+
+      expect(results[0]).to eq(john)
+    end
+  end
+
+  describe "alias index strategy" do
+    subject do
+      Class.new(Elasticity::Document) do
+        configure do |c|
+          c.index_base_name = "users"
+          c.document_type   = "user"
+          c.strategy        =  Elasticity::Strategies::AliasIndex
+
+          c.mapping = {
+            properties: {
+              name: { type: "string" },
+              birthdate: { type: "date" },
+            },
+          }
+        end
+
+        attr_accessor :name, :birthdate
+
+        def to_document
+          { name: name, birthdate: birthdate }
+        end
+      end
+    end
+
+    before do
+      subject.recreate_index
+    end
+
+    after do
+      subject.delete_index
+    end
+
+    it "remaps to a different index transparently" do
+      john = subject.new(name: "John", birthdate: "1985-10-31")
+      mari = subject.new(name: "Mari", birthdate: "1986-09-24")
+
+      john.update
+      mari.update
+
+      subject.flush_index
+
+      results = subject.search(sort: :name)
+      expect(results.total).to eq 2
+
+      subject.remap!
+
+      john.update
+      mari.delete
+
+      subject.flush_index
+
+      results = subject.search(sort: :name)
+      expect(results.total).to eq 1
+
+      expect(results[0]).to eq(john)
+    end
+
+    it "handles in between state while remapping" do
+      docs = 2000.times.map do |i|
+        subject.new(name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+      end
+
+      t = Thread.new { subject.remap! }
+
+      to_update = docs.sample(10)
+      to_delete = (docs - to_update).sample(10)
+
+      to_update.each(&:update)
+      to_delete.each(&:delete)
+
+      20.times.map do |i|
+        subject.new(name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+      end
+
+      t.join
+
+      subject.flush_index
+      results = subject.search(sort: :name)
+      expect(results.total).to eq(2010)
+    end
+
+    it "recover from remap interrupts" do
+      docs = 2000.times.map do |i|
+        subject.new(name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+      end
+
+      t = Thread.new { subject.remap! }
+
+      to_update = docs.sample(10)
+      to_delete = (docs - to_update).sample(10)
+
+      to_update.each(&:update)
+      to_delete.each(&:delete)
+
+      20.times.map do |i|
+        subject.new(name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+      end
+
+      t.raise("Test Interrupt")
+      expect { t.join }.to raise_error("Test Interrupt")
+
+      subject.flush_index
+      results = subject.search(sort: :name)
+      expect(results.total).to eq(2010)
+    end
+  end
+end

data/spec/rspec_config.rb

@@ -1,4 +1,3 @@
-require "elasticity_base"
 require "codeclimate-test-reporter"
 require "simplecov"
 require "oj"
@@ -12,17 +11,21 @@ def elastic_search_client
   @elastic_search_client = Elasticsearch::Client.new host: "http://0.0.0.0:9200"
 end
 
-logger = Logger.new("spec/spec.log", File::WRONLY | File::APPEND | File::CREAT)
+logger = Logger.new("spec/spec.log")
+logger.level = Logger::DEBUG
+
+ActiveSupport::LogSubscriber.logger = logger
+Elasticity::LogSubscriber.attach_to(:elasticity)
 
 RSpec.configure do |c|
   c.disable_monkey_patching!
 
   c.before(:suite) do
-    logger.info "rspec.init Starting test suite execution"
+    logger.info "init.rspec Starting test suite execution"
   end
 
   c.before(:each) do |example|
-    logger.info "rspec.spec #{example.full_description}"
+    logger.info "spec.rspec #{example.full_description}"
 
     if example.metadata[:elasticsearch]
       client = elastic_search_client
@@ -31,7 +34,6 @@ RSpec.configure do |c|
     end
 
     Elasticity.configure do |e|
-      e.logger    = logger
       e.client    = client
       e.namespace = "elasticity_test"
     end

data/spec/units/document_spec.rb

@@ -16,13 +16,12 @@ RSpec.describe Elasticity::Document do
 
   let :klass do
     Class.new(described_class) do
-      # Override the name since this is an anonymous class
-      def self.name
-        "ClassName"
+      configure do |c|
+        c.index_base_name = "class_names"
+        c.document_type   = "class_name"
+        c.mapping         = mappings
       end
 
-      self.mappings = mappings
-
       attr_accessor :name, :items
 
       def to_document
@@ -31,12 +30,12 @@ RSpec.describe Elasticity::Document do
     end
   end
 
-  let :index do
-    double(:index, create_if_undefined: nil, name: "elasticity_test_class_names")
+  let :strategy do
+    double(:strategy)
   end
 
   before :each do
-    allow(Elasticity::Index).to receive(:new).and_return(index)
+    allow(Elasticity::Strategies::SingleIndex).to receive(:new).and_return(strategy)
   end
 
   it "requires subclasses to define to_document method" do
@@ -46,53 +45,45 @@ RSpec.describe Elasticity::Document do
   context "class" do
     subject { klass }
 
-    it "extracts index name and document type from the class name" do
-      expect(subject.namespaced_index_name).to eq "elasticity_test_class_names"
-      expect(subject.document_type).to eq "class_name"
-      expect(subject.index.name).to eq "elasticity_test_class_names"
-    end
-
-    it "have an associated Index instance" do
-      client   = double(:client)
-      settings = double(:settings)
-
-      Elasticity.config.settings = settings
-      Elasticity.config.client   = client
-
-      expect(Elasticity::Index).to receive(:new).with(client, "elasticity_test_class_names").and_return(index)
-
-      expect(subject.index).to be index
+    it "properly instantiate from search hit" do
+      hit = { "_id" => 1, "_source" => { "name" => "foo", "items" => [{ name: "bar" }] }, "highlight" => { "name" => "<em>foo</em>" } }
+      doc = subject.from_hit(hit)
+      expect(doc.name).to eq "foo"
+      expect(doc.items).to eq [{ name: "bar" }]
+      expect(doc.highlighted.name).to eq "<em>foo</em>"
+      expect(doc.highlighted.items).to eq [{ name: "bar" }]
     end
 
     it "searches using DocumentSearch" do
       body   = double(:body)
       search = double(:search)
-      expect(Elasticity::Search).to receive(:new).with(index, "class_name", body).and_return(search)
+
+      expect(strategy).to receive(:search).with("class_name", body).and_return(search)
 
       doc_search = double(:doc_search)
-      expect(Elasticity::DocumentSearchProxy).to receive(:new).with(search, subject).and_return(doc_search)
+      expect(Elasticity::Search::DocumentProxy).to receive(:new).with(search, subject).and_return(doc_search)
 
       expect(subject.search(body)).to be doc_search
     end
 
-    it "gets specific document from the index" do
+    it "gets specific document from the strategy" do
       doc = { "_id" => 1, "_source" => { "name" => "Foo", "items" => [{ "name" => "Item1" }]}}
-      expect(index).to receive(:get_document).with("class_name", 1).and_return(doc)
+      expect(strategy).to receive(:get_document).with("class_name", 1).and_return(doc)
       expect(subject.get(1)).to eq klass.new(_id: 1, name: "Foo", items: [{ "name" => "Item1" }])
     end
 
-    it "deletes specific document from index" do
-      index_ret = double(:index_return)
-      expect(index).to receive(:delete_document).with("class_name", 1).and_return(index_ret)
-      expect(subject.delete(1)).to eq index_ret
+    it "deletes specific document from strategy" do
+      strategy_ret = double(:strategy_return)
+      expect(strategy).to receive(:delete_document).with("class_name", 1).and_return(strategy_ret)
+      expect(subject.delete(1)).to eq strategy_ret
     end
   end
 
   context "instance" do
     subject { klass.new _id: 1, name: "Foo", items: [{ name: "Item1" }] }
 
-    it "stores the document in the index" do
-      expect(index).to receive(:index_document).with("class_name", 1, { name: "Foo", items: [{ name: "Item1" }] })
+    it "stores the document in the strategy" do
+      expect(strategy).to receive(:index_document).with("class_name", 1, { name: "Foo", items: [{ name: "Item1" }] }).and_return("_id" => "1", "created" => true)
       subject.update
     end
   end

data/spec/units/multi_search_spec.rb

@@ -2,10 +2,18 @@ require "elasticity/search"
 require "elasticity/multi_search"
 
 RSpec.describe Elasticity::MultiSearch do
+  let :client do
+    double(:client)
+  end
+
   let :klass do
     Class.new do
       include ActiveModel::Model
-      attr_accessor :_id, :name, :highlighted
+      attr_accessor :_id, :name
+
+      def self.from_hit(hit)
+        new(_id: hit["_id"], name: hit["_source"]["name"])
+      end
 
       def ==(other)
         self._id == other._id && self.name == other.name
@@ -23,8 +31,8 @@ RSpec.describe Elasticity::MultiSearch do
   end
 
   it "performs multi search" do
-    subject.add(:first, Elasticity::Search.new(double(:index, name: "index_first"), "document_first", { search: :first }), documents: klass)
-    subject.add(:second, Elasticity::Search.new(double(:index, name: "index_second"), "document_second", { search: :second }), documents: klass)
+    subject.add(:first, Elasticity::Search::Facade.new(client, Elasticity::Search::Definition.new("index_first", "document_first", { search: :first })), documents: klass)
+    subject.add(:second, Elasticity::Search::Facade.new(client, Elasticity::Search::Definition.new("index_second", "document_second", { search: :second })), documents: klass)
 
     expect(Elasticity.config.client).to receive(:msearch).with(body: [
       { index: "index_first", type: "document_first", search: { search: :first } },

data/spec/units/search_spec.rb

@@ -1,7 +1,8 @@
 require "elasticity/search"
 
 RSpec.describe "Search" do
-  let(:index)          { double(:index) }
+  let(:client)         { double(:client) }
+  let(:index_name)     { "index_name" }
   let(:document_type)  { "document" }
   let(:body)           { {} }
 
@@ -23,16 +24,25 @@ RSpec.describe "Search" do
     { "hits" => { "total" => 0, "hits" => [] }}
   end
 
-  let :highlight_response do
-    { "hits" => { "total" => 1, "hits" => [
-      { "_id" => 1, "_source" => { "name" => "foo", "age" => 21 }, "highlight" => { "name" => "<em>foo</em>" } },
+  let :scan_response do
+    { "_scroll_id" => "abc123", "hits" => { "total" => 2 } }
+  end
+
+  let :scroll_response do
+    { "_scroll_id" => "abc456", "hits" => { "total" => 2, "hits" => [
+      { "_id" => 1, "_source" => { "name" => "foo" } },
+      { "_id" => 2, "_source" => { "name" => "bar" } },
     ]}}
   end
 
   let :klass do
     Class.new do
       include ActiveModel::Model
-      attr_accessor :_id, :name, :age, :highlighted
+      attr_accessor :_id, :name, :age
+
+      def self.from_hit(hit)
+        new(_id: hit["_id"], name: hit["_source"]["name"], age: hit["_source"]["age"])
+      end
 
       def ==(other)
         self._id == other._id && self.name == other.name
@@ -40,13 +50,13 @@ RSpec.describe "Search" do
     end
   end
 
-  describe Elasticity::Search do
+  describe Elasticity::Search::Facade do
     subject do
-      described_class.new(index, document_type, body)
+      described_class.new(client, Elasticity::Search::Definition.new(index_name, document_type, body))
     end
 
     it "searches the index and return document models" do
-      expect(index).to receive(:search).with(document_type, body).and_return(full_response)
+      expect(client).to receive(:search).with(index: index_name, type: document_type, body: body).and_return(full_response)
 
       docs = subject.documents(klass)
       expected = [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")]
@@ -64,43 +74,43 @@ RSpec.describe "Search" do
       expect(Array(docs)).to eq expected
     end
 
+    it "searches using scan&scroll" do
+      expect(client).to receive(:search).with(index: index_name, type: document_type, body: body, search_type: "scan", size: 100, scroll: "1m").and_return(scan_response)
+      expect(client).to receive(:scroll).with(scroll_id: "abc123", scroll: "1m").and_return(scroll_response)
+      expect(client).to receive(:scroll).with(scroll_id: "abc456", scroll: "1m").and_return(empty_response)
+
+      docs = subject.scan_documents(klass)
+      expected = [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")]
+
+      expect(docs.total).to eq 2
+
+      expect(docs).to_not be_empty
+      expect(docs).to_not be_blank
+
+      expect(Array(docs)).to eq expected
+    end
+
     it "searches the index and return active record models" do
-      expect(index).to receive(:search).with(document_type, body.merge(_source: false)).and_return(ids_response)
+      expect(client).to receive(:search).with(index: index_name, type: document_type, body: body.merge(_source: false)).and_return(ids_response)
 
       relation = double(:relation,
         connection: double(:connection),
         table_name: "table_name",
         klass: double(:klass, primary_key: "id"),
+        to_sql: "SELECT * FROM table_name WHERE id IN (1)"
       )
       allow(relation.connection).to receive(:quote_column_name) { |name| name }
 
-      expect(relation).to receive(:where).with(id: [1,2]).and_return(relation)
+      expect(relation).to receive(:where).with("table_name.id IN (?)", [1, 2]).and_return(relation)
       expect(relation).to receive(:order).with("FIELD(table_name.id,1,2)").and_return(relation)
 
-      expect(subject.active_records(relation).mapping).to be relation
-    end
-
-    it "return relation.none from activerecord relation with no matches" do
-      expect(index).to receive(:search).with(document_type, body.merge(_source: false)).and_return(empty_response)
-
-      relation = double(:relation)
-      expect(relation).to receive(:none).and_return(relation)
-
-      expect(subject.active_records(relation).mapping).to be relation
-    end
-
-    it "creates highlighted object for documents" do
-      expect(index).to receive(:search).with(document_type, body).and_return(highlight_response)
-      doc = subject.documents(klass).first
-
-      expect(doc).to_not be_nil
-      expect(doc.highlighted).to eq klass.new(_id: 1, name: "<em>foo</em>", age: 21)
+      expect(subject.active_records(relation).to_sql).to eq "SELECT * FROM table_name WHERE id IN (1)"
     end
   end
 
-  describe Elasticity::DocumentSearchProxy do
+  describe Elasticity::Search::DocumentProxy do
     let :search do
-      Elasticity::Search.new(index, "document", body)
+      Elasticity::Search::Facade.new(client, Elasticity::Search::Definition.new(index_name, "document", body))
     end
 
     subject do
@@ -108,7 +118,7 @@ RSpec.describe "Search" do
     end
 
     it "automatically maps the documents into the provided Document class" do
-      expect(index).to receive(:search).with(document_type, body).and_return(full_response)
+      expect(client).to receive(:search).with(index: index_name, type: document_type, body: body).and_return(full_response)
       expect(Array(subject)).to eq [klass.new(_id: 1, name: "foo"), klass.new(_id: 2, name: "bar")]
     end