elastic_queue 0.0.2

data/lib/elastic_queue.rb

@@ -0,0 +1,7 @@
+require 'active_support'
+require 'elasticsearch'
+require 'elastic_queue/base'
+require 'elastic_queue/queueable'
+
+module ElasticQueue
+end

data/spec/factories.rb

@@ -0,0 +1,64 @@
+FactoryGirl.define do
+  sequence :email_address do |n|
+    "testy#{n}@example.com"
+  end
+
+  sequence :user_id do
+    (10000...10005).to_a.sample
+  end
+
+  sequence :agent_fee_sales_session_status do
+    ['active', 'dead'].sample
+  end
+
+  factory :agent_fee_sales_session do
+    agent
+    status { generate(:agent_fee_sales_session_status) }
+    assigned_to { generate(:user_id) }
+    assigned_at Time.parse('2013-12-02 08:40:33')
+    expires_at Time.parse('2014-06-02 08:40:33')
+    follow_up Time.parse('2013-12-17 06:00:00')
+    hot '1'
+    priority 'high'
+    created_at Time.parse('2013-11-18 14:36:05')
+    updated_at Time.parse('2013-12-17 11:31:08')
+    factory :null_follow_up_agent_fee_sales_session do
+      follow_up nil
+    end
+  end
+
+  factory :agent do
+    after(:build) { |agent| agent.class.skip_callback(:save, :after, :notate_changes) }
+    user
+    company
+    status 'active'
+    name_on_license 'Testy'
+    license_type 'Salesperson'
+    license_number '111111'
+    license_state 'CA'
+    broker_name 'Ali'
+    years_in_real_estate '12 years'
+  end
+
+  factory :company do
+    name 'Flywheel'
+    address '233 Post st.'
+    city 'San Francisco'
+    state 'CA'
+    zip '94104' 
+  end
+
+  factory :user do
+    after(:build) { |user| user.class.skip_callback(:save, :after, :notate_changes) }
+    email { generate(:email_address) }
+    login { |u| u.email }
+    password '1234567'
+    first_name 'Testy'
+    last_name 'Testerson'
+    phone_office '415-555-5555'
+    factory :agent_user do
+      user_type 'agent'
+    end
+  end
+
+end

data/spec/lib/elastic_queue_functional_spec.rb

@@ -0,0 +1,276 @@
+require 'elasticsearch'
+require 'spec_helper'
+require_relative '../../lib/elastic_queue'
+
+describe 'searching:' do
+  before(:all) do
+    class TestQueue < ElasticQueue::Base
+      models :agent_fee_sales_session, :agent
+    end
+    @test_queue = TestQueue
+    AgentFeeSalesSession.send(:include, ElasticQueueable)
+    AgentFeeSalesSession.send(:queue_attributes, :agent_id, :status, :assigned_to_id, :assigned_at, :expires_at, :follow_up, :priority, :hot, :created_at, :updated_at, :first_name, :last_name, :email)
+    Agent.send(:include, ElasticQueueable)
+    Agent.send(:queue_attributes, :status, :license_state)
+    @count = 10
+    @count.times { FactoryGirl.create(:agent_fee_sales_session) }
+    @test_queue.index_models
+    refresh_index
+  end
+
+  after(:all) do
+    AgentFeeSalesSession.all.each(&:delete)
+    Agent.all.each(&:delete)
+    test_search_client.indices.delete index: 'test_queue'
+  end
+
+  describe '#search' do
+
+    it 'returns an ElasticQueueResults object' do
+      expect(@test_queue.search('Testy AND Testerson').class).to be(ElasticQueueResults)
+    end
+
+    it 'searches a search string against default search fields' do
+      a = AgentFeeSalesSession.first
+      a.first_name = 'Zesty'
+      a.save
+      @test_queue.index_models
+      refresh_index
+      expect(@test_queue.search('Testy AND Testerson').count).to be(@count - 1)
+      expect(@test_queue.search('Zesty').count).to be(1)
+    end
+
+    it 'searches a search string against specific search fields' do
+      pending 'yagni'
+    end
+
+    it 'eager loads models when asked to' do
+      expect(@test_queue.search('Testy Testerson', {}, { agent_fee_sales_session: :agent }).first.association(:agent).loaded?).to be(true)
+      expect(@test_queue.search('Testy Testerson', {}, { agent_fee_sales_session: [agent: :user] }).first.agent.association(:user).loaded?).to be(true)
+    end
+
+    it 'does not eager load when not asked to' do
+       expect(@test_queue.search('Testy Testerson').first.association(:agent).loaded?).to be(false)
+    end
+
+    it 'paginates' do
+      expect(@test_queue.search('Testy Testerson', { per_page: 1 }).count).to be(1)
+      first_page = @test_queue.search('Testy Testerson', { per_page: 1, page: 1 })
+      second_page = @test_queue.search('Testy Testerson', { per_page: 1, page: 2 })
+      expect(first_page.first.id != second_page.first.id).to be(true)
+      expect(second_page.results.current_page.to_s).to eq('2')
+    end
+
+    it 'can sort results by a term' do
+      order = @test_queue.search('Testy Testerson', { order_by: :assigned_to_id }).map(&:assigned_to)
+      expect(order).to eq(order.sort)
+    end
+
+    it 'can sort in ascending and decending order' do
+      order_desc = @test_queue.search('Testy Testerson', { order_by: :id, order: :desc }).map(&:id)
+      order_asc = @test_queue.search('Testy Testerson', { order_by: :id, order: :asc }).map(&:id)
+      expect(order_desc.reverse).to eq(order_asc)
+    end
+
+  end
+
+  describe '#queue' do
+
+    describe 'with params hashes' do
+
+      it 'returns all records if not passed any parameters' do
+        expect(@test_queue.queue().length).to be(@count*2)
+      end
+
+      it 'returns models' do
+        expect(@test_queue.queue().first).to be_a(ActiveRecord::Base)
+      end
+
+      describe 'with eager loading' do
+
+        it 'eager loads when asked to' do
+          expect(@test_queue.queue({ model: :agent_fee_sales_session }, { agent_fee_sales_session: :agent }).first.association(:agent).loaded?).to be(true)
+          expect(@test_queue.queue({ model: :agent_fee_sales_session }, { agent_fee_sales_session: [agent: :user] }).first.agent.association(:user).loaded?).to be(true)
+        end
+
+        it 'does not eager load when not asked to' do
+          expect(@test_queue.queue({ model: :agent_fee_sales_session }).first.association(:agent).loaded?).to be(false)
+        end
+
+        it 'properly orders eager loaded models, including across models' do
+          order_with_eager_loading = @test_queue.queue({ model: [:agent_fee_sales_session, :agent], order_by: :id }, { agent_fee_sales_session: :agent, agent: :user }).map(&:id)
+          order_without_eager_loading = @test_queue.queue({ model: [:agent_fee_sales_session, :agent], order_by: :id }).map(&:id)
+          expect(order_with_eager_loading <=> order_without_eager_loading).to be(0)
+        end
+
+      end
+
+      describe 'with queue options' do
+
+        it 'can recover gracefully from bad searches' do
+          expect(@test_queue.queue({ model: :agent_fee_sales_session, sort_by: 'non_attribute', non_attribute: true })).to_not raise_error
+        end
+
+        it 'accepts either strings or symbols' do
+          with_strings = @test_queue.queue({ model: 'agent_fee_sales_session', status: 'active' }).map(&:id)
+          with_symbols = @test_queue.queue({ model: :agent_fee_sales_session, status: :active }).map(&:id)
+          expect(with_symbols).to eq(with_strings)
+        end
+
+        it 'can search across multiple models' do
+          pending
+        end
+
+        it 'can filter by a term' do
+          expect(@test_queue.queue({ status: :active }).map(&:status).uniq).to eq(['active'])
+          expect(@test_queue.queue({ status: :fake_status })).to eq([])
+        end
+
+        it 'can filter by a term even if some of the models it includes don\'t have that term' do
+          pending 'verify specification for this'
+          # agent_id = AgentFeeSalesSession.first.agent_id
+          # license_state = Agent.first.license_state
+          # expect(@test_queue.queue({ agent_id: agent_id }).length).to be(1)
+          # expect(@test_queue.queue({ agent_id: agent_id, license_state: license_state }).length).to be(1)
+        end
+
+        it 'can filter by a range' do
+          the_beginning_of_time = (Date.today - 10.years).to_time
+          agent_fee_sales_session_created_before = Time.now
+          expect(@test_queue.queue({ created_at_between: [the_beginning_of_time, Time.now] }).length).to be(@count)
+          # expect(@test_queue.queued({ created_at_between: [the_beginning_of_time, agent_fee_sales_session_created_before] }).map{ |model| model.class.to_s }.uniq).to eq(['AgentFeeSalesSession'])
+        end
+
+        it 'can take multiple values for one term' do #or filter
+          id_1, id_2 = AgentFeeSalesSession.first(2).map(&:id)
+          expect(@test_queue.queue({ model: :agent_fee_sales_session, id: [id_1, id_2] }).length).to be(2)
+        end
+
+        it 'can filter out a term' do #not filter
+          id_1 = AgentFeeSalesSession.first.id
+          expect(@test_queue.queue({ model: :agent_fee_sales_session, id_not: [id_1] }).length).to be(@count - 1) 
+        end
+
+        describe 'can filter on null values (also tests #null_filter)' do
+          before(:all) do
+            @null_valued_record = FactoryGirl.create(:null_follow_up_agent_fee_sales_session)
+            @test_queue.index_models
+            refresh_index
+          end
+
+          after(:all) do
+            @null_valued_record.delete
+            @test_queue.index_models
+            refresh_index
+          end
+
+          it 'can filter for null values' do #null_filter
+            expect(@test_queue.queue({ model: :agent_fee_sales_session, follow_up_null: true }).length).to be(1)
+          end
+
+          it 'can filter against null values' do #null_filter
+            expect(@test_queue.queue({ model: :agent_fee_sales_session, follow_up_null: false }).length).to be(@count)
+          end
+
+        end
+
+        it 'can sort results by a term' do
+          order = @test_queue.queue({ model: [:agent_fee_sales_session], order_by: :assigned_to_id }).map(&:assigned_to)
+          expect(order).to eq(order.sort)
+        end
+        
+        it 'can sort in ascending and decending order' do
+          order_desc = @test_queue.queue({ model: ['agent_fee_sales_session', 'agent'], order_by: :id, order: :desc }).map(&:id)
+          order_asc = @test_queue.queue({ model: ['agent_fee_sales_session', 'agent'], order_by: :id, order: :asc }).map(&:id)
+          expect(order_desc.reverse).to eq(order_asc)
+        end
+
+        it 'can sort results by multiple terms' do
+          sql_order = AgentFeeSalesSession.order('assigned_to ASC, id DESC').pluck(:id)
+          elastic_queue_order = @test_queue.queue({ model: :agent_fee_sales_session, order_by: [ [:assigned_to_id, :asc], [:id, :desc] ] }).map(&:id)
+          expect(elastic_queue_order).to eq(sql_order)
+        end
+
+      end
+
+    end
+
+    describe 'with chainable methods' do
+
+      describe '#models_include' do
+
+        it 'eager loads models' do
+          expect(@test_queue.model_includes(:agent_fee_sales_session, :agent).queue({ model: :agent_fee_sales_session }).first.association(:agent).loaded?).to be(true)
+        end
+
+      end
+
+      describe '#page, #per_page' do
+        it 'paginates results' do
+          page_1_ids = @test_queue.page(1).per_page(2).queue({ model: :agent_fee_sales_session }).map(&:id)
+          page_2_ids = @test_queue.page(2).per_page(2).queue({ model: :agent_fee_sales_session }).map(&:id)
+          expect(page_1_ids & page_2_ids).to eq([])
+        end
+
+        it 'returns the last page of results if asked for too high of a page number' do
+          expect(@test_queue.page(@count*2).per_page(2).queue({ model: :agent_fee_sales_session })).to_not eq([])
+        end
+
+      end
+
+      describe 'dynamic methods' do
+
+        it 'return the same results as passing an options hash' do
+          chained_dynamic_methods_search = @test_queue.model(:agent_fee_sales_session).order_by([:assigned_to_id, :asc]).order_by([:id, :desc]).queue.map(&:id)
+          options_hash_search = @test_queue.queue({ model: :agent_fee_sales_session, order_by: [ [:assigned_to_id, :asc], [:id, :desc] ] }).map(&:id)
+          expect(chained_dynamic_methods_search).to eq(options_hash_search)
+        end
+
+        it 'isn\'t picky about how you enter array values' do
+          pending 'what do I mean by this?'
+        end
+
+        it 'does not break method_missing' do
+          expect{ @test_queue.fake_method }.to raise_error(NoMethodError)
+        end
+
+        it 'does not break respond_to?' do
+          expect(!@test_queue.respond_to?(:fake_method) && @test_queue.respond_to?(:to_s)).to be(true)
+        end
+
+        it 'provides methods for every queue attribute of an indexed model' do
+          AgentFeeSalesSession.queue_attribute_method_names.each do |attr_name| # relies on elastic_queueable module
+            @test_queue.respond_to?(attr_name) || fail
+          end
+        end
+
+        it 'provides *_before, *_after, *_not, *_null, *_greater_than and *_less_than methods for every queue attribute of an indexed model' do
+          AgentFeeSalesSession.queue_attribute_method_names.each do |attr_name| # relies on elastic_queueable module
+            ['_before', '_after', '_not', '_null', '_greater_than', '_less_than'].each do |suffix|
+              @test_queue.respond_to?("#{attr_name.to_s}#{suffix}".to_sym) || fail
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+  describe '#queue_count' do
+    it 'returns the same number of records as #queue' do
+      expect(@test_queue.queue_count({})).to eq(@test_queue.queue({}).count)
+    end
+
+    it 'works with chainable methods' do
+      pending
+    end
+
+  end
+
+  describe '#queue_with_data' do
+    pending
+  end
+
+end

data/spec/lib/elastic_queue_unit_spec.rb

@@ -0,0 +1,220 @@
+require 'elasticsearch'
+require 'spec_helper'
+require_relative '../../lib/elastic_queue'
+
+describe ElasticQueue do
+
+  before(:all) do
+    class TestQueue < ElasticQueue::Base
+      models :agent_fee_sales_session, :agent
+    end
+    @test_queue = TestQueue
+    AgentFeeSalesSession.send(:include, ElasticQueueable)
+    AgentFeeSalesSession.send(:queue_attributes, :agent_id, :status, :assigned_to_id, :assigned_at, :expires_at, :follow_up, :priority, :hot, :created_at, :updated_at)
+    Agent.send(:include, ElasticQueueable)
+    Agent.send(:queue_attributes, :status, :license_state)
+  end
+
+  describe 'naming:' do
+
+    describe '#index_name' do
+      it 'returns the correct index name' do
+        expect('test_queue') == @test_queue.index_name
+      end
+    end
+
+    describe '#model_names' do #Randomized with seed 41805 will cause this to fail 'cause we already sent models to TestQueue in the next block
+      it 'complains if the user hasn\'t set any models' do
+        class TestQueue < ElasticQueue::Base
+        end
+        expect{TestQueue2.model_names}.to raise_error(NotImplementedError)
+      end
+
+      it 'returns the correct model names' do
+        expect(@test_queue.model_names).to eq([:agent_fee_sales_session, :agent])
+      end
+    end
+
+  end
+
+  describe 'index management' do
+
+    before(:each) do
+      @model = FactoryGirl.create(:agent_fee_sales_session)
+    end
+
+    after(:each) do
+      if test_search_client.indices.exists index: 'test_queue'
+        test_search_client.indices.delete index: 'test_queue'
+      end
+    end
+
+    describe '#index_exists?' do
+
+      it 'returns false for a non-existing index' do
+        class TestQueue < ElasticQueue::Base
+        end
+        expect(TestQueue3.index_exists?).to eq(false)
+      end
+
+      it 'returns true for an existing index' do
+        test_search_client.indices.create index: 'test_queue'
+        expect(@test_queue.index_exists?).to eq(true)
+      end
+
+    end
+
+    describe '#initialize_index' do
+      pending
+    end
+
+    describe '#create_or_recreate_index' do
+      pending
+    end
+
+    describe '#create_index' do
+      pending
+    end
+
+    describe '#delete_index' do
+      pending
+    end
+
+    describe '#index_models' do
+
+      it 'should create the index' do
+        @test_queue.index_models
+        expect(test_search_client.indices.exists index: 'test_queue').to eq(true)
+      end
+
+      it 'should index the models' do
+        @test_queue.index_models
+        refresh_index
+        expect((query_all)['hits']['hits'].first['_source']['model']).to eq('agent_fee_sales_session')
+      end
+
+      it 'should overwrite the old index if it already exists' do
+        pending 'is this really testing what I think it is?'
+        @test_queue.index_models
+        refresh_index
+        first_id = query_all['hits']['hits'].last['_source']['id']
+        @model.delete
+        @model2 = FactoryGirl.create(:agent_fee_sales_session)
+        @test_queue.index_models
+        refresh_index
+        second_id = query_all['hits']['hits'].last['_source']['id']
+        expect(first_id == second_id).to eq(false)
+      end
+
+    end
+
+  end
+
+  describe 'record management' do
+
+    before(:each) do
+      @count = 10
+      @count.times{ FactoryGirl.create(:agent_fee_sales_session) }
+      @test_queue.index_models
+      refresh_index
+      @model = FactoryGirl.create(:agent_fee_sales_session)
+    end
+
+    after(:each) do
+      AgentFeeSalesSession.all.each(&:delete)
+      test_search_client.indices.delete index: 'test_queue'
+    end
+
+    describe '#index_model' do
+
+      it 'should add the model to the index' do
+        before_index = query_all['hits']['hits'].map{ |hit| hit['_source']['id'].to_i }
+        expect(before_index.include?(@model.id)).to be(false) # ensure the test is doing what we expect
+        @test_queue.index_model(@model)
+        refresh_index
+        after_index = query_all['hits']['hits'].map{ |hit| hit['_source']['id'].to_i }
+        expect(after_index.include?(@model.id)).to be(true)
+      end
+
+    end
+
+    describe '#upsert_model' do
+      it 'should update an already existing record' do
+        now = Time.now.to_s
+        record = AgentFeeSalesSession.first
+        record.follow_up = now
+        record.save
+        record.reload
+        @test_queue.upsert_model(record)
+        refresh_index
+        query = { 'filter' => { 'and' => [{ 'term' => { 'model' => 'agent_fee_sales_session' } }, { 'term' => { 'id' => record.id.to_s } }] } }
+        query_index = test_search_client.search index: 'test_queue', body: query.to_json
+        expect(Time.parse(query_index['hits']['hits'].first['_source']['follow_up']).to_s).to eq(record.follow_up.to_s)
+      end
+
+      it 'should insert a non-existing record' do
+        query = { 'filter' => { 'term' => { 'model' => 'agent_fee_sales_session' } } }
+        before_upsert = test_search_client.search index: 'test_queue', size: @count * 2, body: query.to_json
+        @test_queue.upsert_model(@model)
+        refresh_index
+        after_upsert = test_search_client.search index: 'test_queue', size: @count * 2, body: query.to_json
+        expect(before_upsert['hits']['total'] + 1).to be(after_upsert['hits']['total'])
+        expect(after_upsert['hits']['hits'].map{ |hit| hit['_source']['id'].to_i }.include?(@model.id)).to eq(true)
+      end
+    end
+
+    describe '#remove_model' do
+      it 'should remove a given model from the index' do
+        to_remove = AgentFeeSalesSession.first
+        @test_queue.remove_model(to_remove)
+        refresh_index
+        query = { 'filter' => { 'term' => { 'model' => 'agent_fee_sales_session' } } }
+        after_removal = test_search_client.search index: 'test_queue', size: @count, body: query.to_json
+        expect(after_removal['hits']['hits'].map{ |hit| hit['_source']['id'].to_i }.include?(to_remove.id)).to eq(false)
+      end
+    end
+
+  end
+
+  describe 'percolation' do
+
+    before(:each) do
+      @test_queue.index_models
+      refresh_index
+      @model = FactoryGirl.create(:agent_fee_sales_session)
+    end
+
+    after(:each) do
+      AgentFeeSalesSession.all.each(&:delete)
+      test_search_client.indices.delete index: 'test_queue'
+    end
+
+    describe '#register_percolator_query' do
+      pending
+    end
+
+    describe '#reverse_search' do
+      pending
+    end
+
+    describe '#unregister_percolator_query' do
+      pending
+    end
+
+    describe '#list_percolator_queries' do
+      pending
+    end
+
+    describe '#model_in_queue?' do
+      it 'should return true for a model that would be in the queue' do
+        expect(@test_queue.model_in_queue?(@model, { model: :agent_fee_sales_session })).to be(true)
+      end
+
+      it 'should return false for a model that would not be in the queue' do
+        expect(@test_queue.model_in_queue?(@model, { model: :agent })).to be(false)
+      end
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+# This file is copied to spec/ when you run 'rails generate rspec:install'
+require 'rspec'
+require 'rspec/autorun'
+
+require 'factory_girl'
+require File.expand_path('../../lib/elastic_queue', __FILE__)
+require 'support/elastic_queue_helper'
+
+# Requires supporting ruby files with custom matchers and macros, etc,
+# in spec/support/ and its subdirectories.
+# Dir['spec/support/**/*.rb'].each { |f| require f }
+# require_relative '../../lib/elastic_queue'
+
+RSpec.configure do |config|
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+  config.color_enabled = true
+
+  config.include FactoryGirl::Syntax::Methods
+  config.include(ElasticQueueHelper)
+end

data/spec/support/elastic_queue_helper.rb

@@ -0,0 +1,18 @@
+module ElasticQueueHelper
+
+  TEST_SEARCH_CLIENT = Elasticsearch::Client.new
+  
+  def test_search_client
+    TEST_SEARCH_CLIENT
+  end
+
+  def refresh_index
+    # forces the index to refresh itself so the search doesn't happen before the models are done being added to the index
+    TEST_SEARCH_CLIENT.indices.refresh index: 'test_queue'
+  end
+
+  def query_all
+    query = { 'query' => {'match_all' => {} } }.to_json
+    TEST_SEARCH_CLIENT.search index: 'test_queue', body: query, size: 500
+  end
+end