elastic_queue 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b98fbf9af561bf1f2bdaf418941c235e9efb6d5
-  data.tar.gz: 2a2b8517d4cdb107ad9fdb55ace575741ee5854f
+  metadata.gz: 5f6a30f14c2ff7bb0330c854ea5653d295d4f24b
+  data.tar.gz: 6aeca9a70bf31dc9b65b0c9862dc835c45535ca2
 SHA512:
-  metadata.gz: 6ebcfd62bf72d135e4fa2532d61237996698a8974b99cf2bdfb9293a6705c0f52dc23af9aa6b2e6e46f861d49a711699c5c9b753b9b96c45c72e2c6c5ddcb035
-  data.tar.gz: e7a20b9650fbd4e411e6449d8373a32416c9a9af9257aeec5caa16827a4faa16985d072d132ba7e5be25893a5d104debace1d5385e2fbf7d684961116b4e59f8
+  metadata.gz: 71487d775f4a86d6f9f17abb25173d671b7a153769bdf906e4bd100deede733efec86c85235388e84954a39239448d3afdef7d5c9d5a801dcbd48ab0f47ff990
+  data.tar.gz: 4e17e151ca7cb3885eea1f496e283d5cd0f4c6532b1d56d67c34469bd4205875eb071c1322c3bc712ba6ca7e1d36e86e9b8bd8fc8269ea586b369c6da007641d

data/.gitignore

@@ -17,3 +17,4 @@ tmp
 .yardoc
 _yardoc
 doc/
+spec/eq_test.db

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    elastic_queue (0.0.10)
+    elastic_queue (0.0.11)
+      activerecord (~> 3.0)
       activesupport (~> 3.0)
       elasticsearch (~> 0.4)
       will_paginate (~> 3.0)
@@ -9,9 +10,26 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (3.2.17)
+    activemodel (3.2.18)
+      activesupport (= 3.2.18)
+      builder (~> 3.0.0)
+    activerecord (3.2.18)
+      activemodel (= 3.2.18)
+      activesupport (= 3.2.18)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activesupport (3.2.18)
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
+    arel (3.0.3)
+    builder (3.0.4)
+    columnize (0.8.9)
+    debugger (1.6.6)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.3.2)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.3.2)
     diff-lcs (1.2.5)
     elasticsearch (0.4.11)
       elasticsearch-api (= 0.4.11)
@@ -26,7 +44,7 @@ GEM
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
     i18n (0.6.9)
-    multi_json (1.9.2)
+    multi_json (1.10.0)
     multipart-post (2.0.0)
     rake (10.3.1)
     rspec (2.14.1)
@@ -38,15 +56,18 @@ GEM
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.6)
     sqlite3 (1.3.9)
+    tzinfo (0.3.39)
     will_paginate (3.0.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  activesupport (~> 3.0)
   bundler (~> 1.0)
+  debugger (~> 1.6)
   elastic_queue!
   factory_girl (~> 4.0)
   rake (~> 10.0)
   rspec (~> 2.6)
-  sqlite3
+  sqlite3 (~> 1.3)

data/elastic_queue.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
   s.test_files = Dir['spec/**/*.rb']
 
   s.add_runtime_dependency 'activesupport', '~> 3.0'
+  s.add_runtime_dependency 'activerecord', '~>3.0'
   s.add_runtime_dependency 'elasticsearch', '~> 0.4'
   s.add_runtime_dependency 'will_paginate', '~> 3.0'
   s.add_development_dependency 'bundler', '~> 1.0'
@@ -26,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'factory_girl', '~> 4.0'
   s.add_development_dependency 'sqlite3', '~> 1.3'
   s.add_development_dependency 'rake', '~> 10.0'
-end
+  s.add_development_dependency 'debugger', '~>1.6'
+end

data/lib/elastic_queue/base.rb

@@ -9,7 +9,7 @@ module ElasticQueue
     include Percolation
 
     def self.search_client
-      Elasticsearch::Client.new hosts: ElasticQueue::OPTIONS[:elasticsearch_hosts]
+      @search_client ||= Elasticsearch::Client.new hosts: ElasticQueue::OPTIONS[:elasticsearch_hosts]
     end
 
     def self.models(*models)

data/lib/elastic_queue/filters.rb

@@ -12,18 +12,27 @@ module ElasticQueue
     private
 
     def option_to_filter(key, value)
-      if value.is_a? Array
+      # return and_options(value) if key == :and
+      if [:or, :and].include?(key)
+        join_options(key, value)
+      elsif value.is_a? Array
         or_filter(key, value)
       elsif value.is_a? Hash
-        # date?
-        time_filter(key, value)
+        comparison_filter(key, value)
       elsif value.nil?
+        # e.g. name: nil
         null_filter(key, value)
       else
+        # e.g. status: 'fresh'
         term_filter(key, value)
       end
     end
 
+    def join_options(operator, options)
+      conditions = options.map { |o| options_to_filters(o) }.flatten
+      { operator => conditions }
+    end
+
     def or_filter(term, values)
       # flatten here because ranges return arrays
       conditions = values.map { |v| option_to_filter(term, v) }.flatten
@@ -35,7 +44,7 @@ module ElasticQueue
     end
 
     # take something like follow_up: { before: 'hii', after: 'low' }
-    def time_filter(term, value)
+    def comparison_filter(term, value)
       value.map do |k, v|
         comparator = k.to_sym.in?([:after, :greater_than, :gt]) ? :gt : :lt
         range_filter(term, v, comparator)

data/lib/elastic_queue/persistence.rb

@@ -13,7 +13,7 @@ module ElasticQueue
       end
 
       def create_index
-        search_client.indices.create index: index_name
+        search_client.indices.create index: index_name, body: default_index_settings
         add_mappings
       end
 
@@ -26,12 +26,14 @@ module ElasticQueue
         search_client.indices.refresh index: index_name
       end
 
-      def bulk_index(batch_size = 10_000)
+      # you can pass scopes into bulk_index to be used when fetching records
+      #  bulk_index(scopes: { some_model: [:scope1, :scope2], some_other_model: [:scope3] }) will fetch SomeModel.scope1.scope2 and SomeOtherModel.scope3 and index only those records.
+      def bulk_index(scopes: {}, batch_size: 10_000)
         create_index unless index_exists?
         model_classes.each do |klass|
           # modelclass(model).includes(associations_for_index(model)).
           index_type = klass.to_s.underscore
-          klass.find_in_batches(batch_size: batch_size) do |batch|
+          scoped_class(klass, scopes).find_in_batches(batch_size: batch_size) do |batch|
             body = []
             batch.each do |instance|
               body << { index: { _index: index_name, _id: instance.id, _type: index_type, data: instance.indexed_for_queue } }
@@ -41,9 +43,34 @@ module ElasticQueue
         end
       end
 
+      def scoped_class(klass, scopes)
+        return klass unless scopes[klass.to_s.underscore.to_sym]
+        scopes[klass.to_s.underscore.to_sym].each do |scope|
+          klass = klass.send(scope)
+        end
+        klass
+      end
+
+      def default_index_settings
+        {
+          settings: {
+            analysis: {
+              analyzer: {
+                default: {
+                  type: :custom,
+                  tokenizer: :whitespace,
+                  filter: [:lowercase]
+                }
+              }
+            }
+          }
+        }
+      end
+
       def add_mappings
         model_classes.each do |klass|
-          search_client.indices.put_mapping index: index_name, type: klass.to_s.underscore, body: klass.queue_mapping
+          mapping = klass.queue_mapping
+          search_client.indices.put_mapping index: index_name, type: klass.to_s.underscore, body: mapping if mapping.present?
         end
       end
 

data/lib/elastic_queue/query.rb

@@ -45,40 +45,39 @@ module ElasticQueue
 
     def paginate(options = {})
       options.each { |k, v| @options.send("#{k}=", v) }
-      all.paginate
+      Results.new(@queue, execute(paginate: true), @options).paginate
     end
 
+    # TODO: remove if not using, add per_page if using
     def page=(page)
       @options.page = (page)
     end
 
     def all
-      @results ||= Results.new(@queue, execute, @options)
+      Results.new(@queue, execute, @options).instantiated_queue_items
     end
 
     # return just the ids of the records (useful when combined with SQL queries)
     def ids
-      results = execute
-      results[:hits][:hits].map { |h| h[:_source][:id] }
+      execute[:hits][:hits].map { |h| h[:_source][:id] }
     end
 
     def count
-      res = execute(count: true)
-      res[:hits][:total].to_i
+      execute(count: true, paginate: false)[:hits][:total].to_i
     end
 
-    def execute(count: false)
+    private
+
+    def execute(count: false, paginate: false)
       begin
-        search = execute_query(count: false)
-        search = substitute_page(search) if !count && search['hits']['hits'].length == 0 && search['hits']['total'] != 0
+        search = paginate ? execute_paginated_query : execute_all_query( count: count )
+        search = substitute_page(search) if paginate && !count && search['hits']['hits'].length == 0 && search['hits']['total'] != 0
       rescue Elasticsearch::Transport::Transport::Errors::BadRequest
         search = failed_search
       end
       search.with_indifferent_access
     end
 
-    private
-
     # this allows you to chain scopes
     # the 2+ scopes in the chain will be called
     # on a query object and not on the base object
@@ -89,16 +88,21 @@ module ElasticQueue
       end
     end
 
-    def execute_query(count: false)
-      search_type = count ? 'count' : 'query_then_fetch'
-      @queue.search_client.search index: @queue.index_name, body: body, search_type: search_type, from: @options.from, size: @options.per_page
+    def execute_all_query(count: false)
+      record_count = @queue.search_client.search index: @queue.index_name, body: body, search_type: 'count'
+      return record_count if count
+      @queue.search_client.search index: @queue.index_name, body: body, search_type: 'query_then_fetch', from: 0, size: record_count['hits']['total'].to_i
+    end
+
+    def execute_paginated_query
+      @queue.search_client.search index: @queue.index_name, body: body, search_type: 'query_then_fetch', from: @options.from, size: @options.per_page
     end
 
     def substitute_page(search)
       total_hits = search['hits']['total'].to_i
       per_page = @options.per_page
       @options.page = (total_hits / per_page.to_f).ceil
-      execute_query
+      execute_paginated_query
     end
 
     def failed_search

data/lib/elastic_queue/queueable.rb

@@ -9,6 +9,7 @@ module ElasticQueue
     end
 
     module ClassMethods
+
       def queues(*queues)
         @queues ||= queues
       end

data/lib/elastic_queue/results.rb

@@ -2,9 +2,8 @@ require 'will_paginate/collection'
 
 module ElasticQueue
   class Results
-    attr_reader :paginate
 
-    delegate :empty?, :each, :total_entries, :total_pages, :current_page, to: :paginate
+    attr_reader :instantiated_queue_items
 
     def initialize(queue, search_results, query_options)
       @queue = queue
@@ -12,19 +11,22 @@ module ElasticQueue
       @start = query_options.page
       @per_page = query_options.per_page
       @total = search_results[:hits][:total]
-      @paginate = WillPaginate::Collection.create(@start, @per_page, @total) do |pager|
+    end
+
+    def paginate
+      WillPaginate::Collection.create(@start, @per_page, @total) do |pager|
         pager.replace(@instantiated_queue_items)
       end
     end
 
+    private
+
     def instantiate_queue_items(search_results)
       grouped_results, sort_order = group_sorted_results(search_results)
       records = fetch_records(grouped_results)
       sort_records(records, sort_order)
     end
 
-    private
-
     # group the results by { class_name: [ids] } and save their sorted order
     def group_sorted_results(search_results)
       grouped_results = {}

data/lib/elastic_queue/version.rb

@@ -1,3 +1,3 @@
 module ElasticQueue
-  VERSION = '0.0.10'
+  VERSION = '0.0.11'
 end

data/spec/factories.rb

@@ -1,64 +0,0 @@
-FactoryGirl.define do
-  sequence :email_address do |n|
-    "testy#{n}@example.com"
-  end
-
-  sequence :user_id do
-    (10000...10005).to_a.sample
-  end
-
-  sequence :agent_fee_sales_session_status do
-    ['active', 'dead'].sample
-  end
-
-  factory :agent_fee_sales_session do
-    agent
-    status { generate(:agent_fee_sales_session_status) }
-    assigned_to { generate(:user_id) }
-    assigned_at Time.parse('2013-12-02 08:40:33')
-    expires_at Time.parse('2014-06-02 08:40:33')
-    follow_up Time.parse('2013-12-17 06:00:00')
-    hot '1'
-    priority 'high'
-    created_at Time.parse('2013-11-18 14:36:05')
-    updated_at Time.parse('2013-12-17 11:31:08')
-    factory :null_follow_up_agent_fee_sales_session do
-      follow_up nil
-    end
-  end
-
-  factory :agent do
-    after(:build) { |agent| agent.class.skip_callback(:save, :after, :notate_changes) }
-    user
-    company
-    status 'active'
-    name_on_license 'Testy'
-    license_type 'Salesperson'
-    license_number '111111'
-    license_state 'CA'
-    broker_name 'Ali'
-    years_in_real_estate '12 years'
-  end
-
-  factory :company do
-    name 'Flywheel'
-    address '233 Post st.'
-    city 'San Francisco'
-    state 'CA'
-    zip '94104' 
-  end
-
-  factory :user do
-    after(:build) { |user| user.class.skip_callback(:save, :after, :notate_changes) }
-    email { generate(:email_address) }
-    login { |u| u.email }
-    password '1234567'
-    first_name 'Testy'
-    last_name 'Testerson'
-    phone_office '415-555-5555'
-    factory :agent_user do
-      user_type 'agent'
-    end
-  end
-
-end

data/spec/lib/integration/elastic_queue_base_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe 'ElasticQueue::Base integration' do
+  describe '#search_client' do
+    it 'returns an Elasticsearch::Transport::Client' do
+      expect(ElasticQueue::Base.search_client).to be_a Elasticsearch::Transport::Client
+    end
+    
+    it 'returns the same client every time' do
+      expect(ElasticQueue::Base.search_client.object_id).to eq ElasticQueue::Base.search_client.object_id
+    end
+  end
+
+  describe '#models, also tests(#tell_models, #model_names, #model_classes)' do
+    it '#models sets @models and tells the model about itself' do
+      class Cannibal < ActiveRecord::Base
+        include ElasticQueue::Queueable
+      end
+      # Cannibal.stub(:add_queue)
+      # Cannibal.should_receive(:add_queue).with(:"elastic_queue/base")
+      ElasticQueue::Base.models(:cannibal)
+      expect(ElasticQueue::Base.instance_variable_get('@models')).to eq [:cannibal]
+    end
+  end
+
+  describe '#index_name, #index_name =' do
+    pending('trivial')
+  end
+
+  describe '#eager_load' do
+    pending
+  end
+
+  describe '#eager_loads' do
+    pending
+  end
+
+  describe '#scopes' do
+    pending
+  end
+
+  describe '#scopes' do
+    pending
+  end
+
+  describe '#default_scope' do
+    pending
+  end
+
+  describe '#query' do
+    pending
+  end
+
+  describe '#filter' do
+    pending
+  end
+ 
+  describe '#count' do
+    pending
+  end
+
+  describe '#paginate' do
+    pending('not implemented yet')
+  end
+
+  describe 'instance #query' do
+    pending
+  end
+
+end

data/spec/lib/integration/elastic_queue_filters_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+
+describe 'ElasticQueue::Filters integration' do
+  before :all do
+    class Animal < ActiveRecord::Base
+      include ElasticQueue::Queueable
+      queues :test_animals_queue
+      queue_attributes :dangerous, :cute, :birthdate, :name
+      not_analyzed_queue_attributes :species, :description
+    end
+
+    class TestAnimalsQueue < ElasticQueue::Base
+      models :animal
+    end
+
+    TestAnimalsQueue.create_index
+    
+    @create_animals = -> {
+      Animal.create({ name: 'a', birthdate: Date.today.at_midnight - 1.year })
+      Animal.create({ name: 'b', birthdate: Date.today.at_midnight - 2.years })
+      Animal.create({ name: 'c', birthdate: Date.today.at_midnight - 3.years })
+    }
+  end
+
+  after :all do
+    [:Animal, :TestAnimalsQueue].each do |constant|
+      Object.send(:remove_const, constant)
+    end
+    delete_index('test_animals_queue')
+  end
+
+  describe 'ElasticQueue::Query#filter' do
+    after :each do
+      Animal.all.each(&:destroy)
+    end
+
+    it 'can filter on one value' do
+      @create_animals.call
+      expect(TestAnimalsQueue.query.filter(name: 'a').all.map(&:name)).to eq ['a']
+    end
+
+    it 'can filter by a less than or greater than a time' do
+      @create_animals.call
+      expect(TestAnimalsQueue.query.filter(birthdate: { after: Date.today - 1.year - 1.day }).all.map(&:name)).to eq ['a']
+      expect(TestAnimalsQueue.query.filter(birthdate: { before: Date.today - 2.years - 1.day }).all.map(&:name)).to eq ['c']
+    end
+
+    it 'can filter by a less than and greater than a time' do
+      @create_animals.call
+      expect(TestAnimalsQueue.query.filter(birthdate: { after: Date.today - 2.year - 1.day, before: Date.today - 1.year - 1.day}).all.map(&:name)).to eq ['b']
+    end
+
+    it 'can filter by less than or greater than a string' do
+      @create_animals.call
+      expect(TestAnimalsQueue.query.filter(name: { after: 'a', before: 'c' }).all.map(&:name)).to eq ['b']
+    end
+
+    it 'doesn\'t error if you try to filter on an nonexistent value' do
+      @create_animals.call
+      expect(TestAnimalsQueue.query.filter(likes_peanut_butter: true).all.map(&:name)).to eq []
+    end
+
+    it 'filters underscored values as one word' do
+      Animal.create({ name: 'pin_head' })
+      Animal.create({ name: 'pin' })
+      expect(TestAnimalsQueue.query.filter(name: 'pin').all.map(&:name)).to eq ['pin']
+    end
+
+    it 'treats parentheses as letters' do
+      Animal.create({ name: 'Sr. Honks-a-lot' , species: '(Silly) Goose' })
+      expect(TestAnimalsQueue.query.filter(species: 'Goose').all.map(&:name)).to eq []
+      expect(TestAnimalsQueue.query.filter(species: '(Silly) Goose').all.map(&:name)).to eq ['Sr. Honks-a-lot']
+    end
+
+    it 'automatically joins multiple filter values with an OR' do
+      @create_animals.call
+      expect(TestAnimalsQueue.query.filter(name: ['a', 'b']).all.map(&:name).sort).to eq ['a', 'b']
+    end
+
+    it 'defaults to joining multiple filter keys with an AND' do
+      Animal.create({ name: 'x', species: 'dog' })
+      Animal.create({ name: 'y', species: 'dog' })
+      expect(TestAnimalsQueue.query.filter(name: 'x').filter(species: 'dog').all.map(&:name)).to eq ['x']
+    end
+
+    it 'can join multiple filter keys with an OR' do
+      Animal.create({ name: 'x', species: 'dog' })
+      Animal.create({ name: 'y', species: 'cat' })
+      Animal.create({ name: 'z', species: 'rat' })
+      expect(TestAnimalsQueue.query.filter(or: [{ name: 'x' }, { species: 'cat' }]).all.map(&:name).sort).to eq ['x', 'y']
+    end
+
+    it 'can join multiple filter values with multiple filter keys with an OR' do
+      Animal.create({ name: 'x', species: 'dog' })
+      Animal.create({ name: 'y', species: 'cat' })
+      Animal.create({ name: 'z', species: 'chicken' })
+      Animal.create({ name: 'a', species: 'chicken' })
+      expect(TestAnimalsQueue.query.filter(or: [{ name: 'z' }, { species: ['cat', 'dog'] }]).all.map(&:name).sort).to eq ['x', 'y', 'z']
+    end
+
+    it 'can nest multiple ands and ors' do
+      Animal.create({ name: 'rusty', species: 'dog', dangerous: false })
+      Animal.create({ name: 'killer', species: 'mountain lion', dangerous: true })
+      Animal.create({ name: 'cock-a-doodle-doo', species: 'chicken', dangerous: false })
+      Animal.create({ name: 'old bess', species: 'cow', dangerous: true })
+      Animal.create({ name: 'speedy', species: 'horse', dangerous: false })
+      expect(TestAnimalsQueue.query.filter({
+        # (rusty, killer, speedy) && ( rusty || lucky, killer, old bess, cock-a-doodle-doo)
+        name: ['rusty', 'killer', 'speedy'], #(rusty, killer, speedy) && (
+        or: [
+            and: [ # rusty ||
+              { species: ['dog', 'mountain lion'] },
+              { dangerous: false }
+            ],
+            or: [ #speedy, killer, old bess, cock-a-doodle-doo )
+              { species: ['horse', 'chicken'] },
+              { dangerous: true }
+            ]
+          ]
+      }).all.map(&:name).sort).to eq ['killer', 'rusty', 'speedy']
+    end
+  end
+end