elastic_queue 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5fba6995c9fb76fcb3b2f7e0e204bf5c955e243b
+  data.tar.gz: a47b1f04bf521b2a4c42f1fee5f2be4feee51256
+SHA512:
+  metadata.gz: f7e36b7ac21298dd00e897b1f7c4e16f02a0b20c3bd21b33d4e68fded773acb9ebd712dba768bd73ea216f2f3b52ccc85989168997c27f57dd821f6b8044866e
+  data.tar.gz: 16409af91a7ae6020d5844a68b4ceddfbe0a53283d1bb3e7cb066210d400c40e0963ba7082a0ecaa17700a46cade6ecaa404ee585f6063dd39675c8895b23b1c

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.DS_Store
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/.rubocop.yml

@@ -0,0 +1,20 @@
+# This is the configuration rubocop, a ruby source code linter and checker.
+Documentation:
+  Description: 'Document classes and non-namespace modules.'
+  Enabled: false
+
+FinalNewline:
+  Description: 'Checks for a final newline in a source file.'
+  Enabled: false
+
+LineLength:
+  Description: 'Limit lines to 79 characters.'
+  Enabled: false
+
+MethodLength:
+  Description: 'Avoid methods longer than 10 lines of code.'
+  Enabled: false
+
+RescueModifier:
+  Description: 'Avoid using rescue in its modifier form.'
+  Enabled: false

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# dependencies in elastic-queue.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,56 @@
+PATH
+  remote: .
+  specs:
+    elastic-queue (0.0.2)
+      activesupport
+      elasticsearch
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.0.1)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    atomic (1.1.14)
+    diff-lcs (1.2.5)
+    elasticsearch (0.4.1)
+      elasticsearch-api (= 0.4.1)
+      elasticsearch-transport (= 0.4.1)
+    elasticsearch-api (0.4.1)
+      multi_json
+    elasticsearch-transport (0.4.1)
+      faraday
+      multi_json
+    factory_girl (4.2.0)
+      activesupport (>= 3.0.0)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    i18n (0.6.9)
+    minitest (4.7.5)
+    multi_json (1.8.2)
+    multipart-post (1.2.0)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    thread_safe (0.1.3)
+      atomic
+    tzinfo (0.3.38)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.0.0)
+  elastic-queue!
+  factory_girl
+  rake
+  rspec (~> 2.6)

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 RuthThompson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,12 @@
+# ElasticQueue
+
+[![Code Climate](https://codeclimate.com/github/RuthThompson/elastic_queue.png)](https://codeclimate.com/github/RuthThompson/elastic_queue)
+
+A queueing system built on top of elasticsearch.
+
+### Usage
+TODO
+
+### How it Works
+
+TODO

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'bundler'
+require 'rake'
+require 'rspec/core/rake_task'
+Bundler::GemHelper.install_tasks
+
+desc 'Default: run the specs and features.'
+task :default do
+  system('bundle exec rake -s appraisal spec:unit spec:acceptance features;')
+end
+
+namespace :spec do
+  desc 'Run unit specs'
+  RSpec::Core::RakeTask.new('unit') do |t|
+    t.pattern = 'spec/{*_spec.rb,factory_girl/**/*_spec.rb}'
+  end
+
+  desc 'Run acceptance specs'
+  RSpec::Core::RakeTask.new('acceptance') do |t|
+    t.pattern = 'spec/acceptance/**/*_spec.rb'
+  end
+end
+
+desc 'Run the unit and acceptance specs'
+task :spec => ['spec:unit', 'spec:acceptance']

data/elastic_queue.gemspec

@@ -0,0 +1,28 @@
+require './lib/elastic_queue/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'elastic_queue'
+  s.version     = ElasticQueue::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.summary     = 'A queueing system built on top of elasticsearch.'
+  s.description = 'A library for storing and filtering documents on elastic search with a queue paradigm for retrieval.'
+  s.license     = 'MIT'
+
+  s.required_ruby_version = '>= 1.9.2'
+
+  s.authors   = ['Ruth Thompson', 'Rob Law']
+  s.email    = %w[ ruth@flywheelnetworks.com rob@flywheelnetworks.com ]
+  s.homepage = 'https://github.com/RuthThompson/elastic_queue'
+
+  s.require_paths = %w[ lib ]
+  s.files = `git ls-files`.split("\n")
+  s.test_files = Dir['spec/**/*.rb']
+
+  s.add_dependency 'activesupport'
+  s.add_dependency 'elasticsearch'
+  s.add_dependency 'will_paginate'
+  s.add_development_dependency 'bundler', '>= 1.0.0'
+  s.add_development_dependency 'rspec', '~> 2.6'
+  s.add_development_dependency 'factory_girl'
+  s.add_development_dependency 'rake'
+end

data/lib/elastic_queue/base.rb

@@ -0,0 +1,52 @@
+require 'elasticsearch'
+require 'elastic_queue/persistence'
+require 'elastic_queue/query'
+
+module ElasticQueue
+  class Base
+    include Persistence
+    # include Percolation
+
+    def self.search_client
+      Elasticsearch::Client.new
+    end
+
+    def self.models(*models)
+      @models = models
+    end
+
+    def self.model_names
+      raise NotImplementedError, "No models defined in #{self.class}" unless defined?(@models)
+      @models
+    end
+
+    def self.model_classes
+      model_names.map { |s| s.to_s.camelize.constantize }
+    end
+
+    def self.index_name
+      @index_name ||= to_s.underscore
+    end
+
+    def self.eager_load(includes)
+      @eager_loads = includes
+    end
+
+    def self.eager_loads
+      @eager_loads
+    end
+
+    def self.query(options = {})
+      Query.new(self, options)
+    end
+
+    def self.filter(options)
+      query.filter(options)
+    end
+
+    def self.count
+      query.count
+    end
+
+  end
+end

data/lib/elastic_queue/filters.rb

@@ -0,0 +1,47 @@
+module ElasticQueue
+  module Filters
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+    end
+
+    def options_to_filters(options)
+      options.map { |k, v| option_to_filter(k, v) }.flatten
+    end
+
+    private
+
+    def option_to_filter(key, value)
+      if value.is_a? Array
+        or_filter(key, value)
+      elsif value.is_a? Hash
+        # date?
+        time_filter(key, value)
+      else
+        term_filter(key, value)
+      end
+    end
+
+    def or_filter(term, values)
+      { or: values.map { |v| term_filter(term, v) } }
+    end
+
+    def term_filter(term, value)
+      { term: { term => value } }
+    end
+
+    # take something like follow_up: { before: 'hii', after: 'low' }
+    def time_filter(term, value)
+      value.map do |k, v|
+        comparator = k.to_sym.in?([:after, :greater_than, :gt]) ? :gt : :lt
+        range_filter(term, v, comparator)
+      end
+    end
+
+    # like term filter but for comparison queries
+    def range_filter(term, value, comparator)
+      { range: { term => { comparator => value } } }
+    end
+
+  end
+end

data/lib/elastic_queue/percolation.rb

@@ -0,0 +1,42 @@
+module ElasticQueue
+  module Percolation
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+
+    #   def percolator_queries
+    #     queries = {}
+    #     query = { 'query' => {'match_all' => {}} }
+    #     search = search_client.search index: '_percolator', body: query.to_json, size: 1000
+    #     debugger
+    #     search['hits']['hits'].each  { |hit|           queries[hit['_id']] = hit['_source'] }
+    #     queries
+    #   end
+
+    end
+
+    # def reverse_search(instance)
+    #   body = { 'doc' => instance.indexed_for_queue }
+    #   search_client.percolate index: index_name, body: body.to_json
+    # end
+
+    # def register_percolator_query(name, opts)
+    #   search_client.index index: '_percolator', type: index_name, id: name, body: translate_opts_to_query_for_percolator(opts)
+    # end
+
+    # def unregister_percolator_query(name)
+    #   SEARCH_CLIENT.delete index: '_percolator', type: index_name, id: name
+    # end
+
+    # def model_in_queue?(model, queue_opts)
+    #   return false unless model_names.include?(model.class.to_s.underscore.to_sym)
+    #   search_id = SecureRandom.uuid
+    #   body = { 'doc' => model.indexed_for_queue, 'query' => {'term' => {'_id' => search_id}} }
+    #   SEARCH_CLIENT.index index: '_percolator', type: 'dynamic_percolator', id: search_id, body: translate_opts_to_query_for_percolator(queue_opts), refresh: true
+    #   search = SEARCH_CLIENT.percolate index: 'dynamic_percolator', body: body.to_json
+    #   SEARCH_CLIENT.delete index: '_percolator', type: 'dynamic_percolator', id: search_id
+    #   search['matches'].length == 1
+    # end
+
+  end
+end

data/lib/elastic_queue/persistence.rb

@@ -0,0 +1,68 @@
+# TODO: might want to move these to an Index class
+module ElasticQueue
+  module Persistence
+    extend ActiveSupport::Concern
+    module ClassMethods
+
+      def index_exists?
+        search_client.indices.exists index: index_name
+      end
+
+      def reset_index
+        delete_index if index_exists?
+        create_index
+      end
+
+      def create_index
+        search_client.indices.create index: index_name
+        add_mappings
+      end
+
+      def delete_index
+        search_client.indices.delete index: index_name
+      end
+
+      # not using it, but it is nice for debugging
+      def refresh_index
+        search_client.indices.refresh index: index_name
+      end
+
+      def bulk_index(batch_size = 10_000)
+        create_index unless index_exists?
+        model_classes.each do |klass|
+          # modelclass(model).includes(associations_for_index(model)).
+          index_type = klass.to_s.underscore
+          klass.find_in_batches(batch_size: batch_size) do |batch|
+            body = []
+            batch.each do |instance|
+              body << { index: { _index: index_name, _id: instance.id, _type: index_type, data: instance.indexed_for_queue } }
+            end
+            search_client.bulk body: body
+          end
+        end
+      end
+
+      def add_mappings
+        model_classes.each do |klass|
+          search_client.indices.put_mapping index: index_name, type: klass.to_s.underscore, body: klass.queue_mapping
+        end
+      end
+
+      # TODO: move these to an instance?
+      def index_model(instance)
+        search_client.index index: index_name, id: instance.id, type: instance.class.to_s.underscore, body: instance.indexed_for_queue
+      end
+
+      def upsert_model(instance)
+        body = { doc: instance.indexed_for_queue, doc_as_upsert: true }
+        search_client.update index: index_name, id: instance.id, type: instance.class.to_s.underscore, body: body, refresh: true
+      end
+
+      def remove_model(instance)
+        search_client.delete index: index_name, id: instance.id, type: instance.class.to_s.underscore
+      end
+
+    end
+
+  end
+end

data/lib/elastic_queue/query.rb

@@ -0,0 +1,62 @@
+require 'elastic_queue/query_options'
+require 'elastic_queue/results'
+
+module ElasticQueue
+  class Query
+
+    def initialize(queue, options = {})
+      @queue = queue
+      @options = QueryOptions.new(options)
+    end
+
+    def filter(options)
+      @options.add_filter(options)
+      self
+    end
+
+    def filters
+      @options.filters
+    end
+
+    def sort(options)
+      @options.add_sort(options)
+      self
+    end
+
+    def sorts
+      @options.sorts
+    end
+
+    def paginate(options = {})
+      options.each { |k, v| @options.send("#{k}=", v) }
+      all.paginate
+    end
+
+    def all
+      @results ||= Results.new(@queue, execute, @options)
+    end
+
+    def count
+      res = execute(count: true)
+      res[:hits][:total].to_i
+    end
+
+    def execute(count: false)
+      search_type = count ? 'count' : 'query_then_fetch'
+      begin
+        search = @queue.search_client.search index: @queue.index_name, body: @options.body, search_type: search_type, from: @options.from, size: @options.per_page
+        # search[:page] = @page
+        # search = substitute_page(opts, search) if !count && opts[:page_substitution_ok] && search['hits']['hits'].length == 0 && search['hits']['total'] != 0
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        search = failed_search
+      end
+      search.with_indifferent_access
+    end
+
+    def failed_search
+      { page: 0, hits: { hits: [], total: 0 } }
+    end
+
+  end
+
+end

data/lib/elastic_queue/query_options.rb

@@ -0,0 +1,48 @@
+require 'elastic_queue/filters'
+require 'elastic_queue/sorts'
+
+module ElasticQueue
+  class QueryOptions
+    include Filters
+    include Sorts
+
+    attr_reader :filters, :sorts
+    attr_accessor :per_page
+
+    def initialize(options = {})
+      @options = { per_page: 30, page: 1 }.merge(options)
+      @filters = { and: [] }.with_indifferent_access
+      @sorts = []
+      self.per_page = @options[:per_page]
+      self.page = @options[:page]
+    end
+
+    def add_filter(options)
+      @filters[:and] += options_to_filters(options)
+    end
+
+    def add_sort(options)
+      @sorts += options_to_sorts(options)
+    end
+
+    def from
+      (page - 1) * per_page
+    end
+
+    def page=(num)
+      @page = num.to_i unless num.blank?
+    end
+
+    def page
+      @page
+    end
+
+    def body
+      b = {}
+      b[:filter] = @filters unless @filters[:and].blank?
+      b[:sort] = @sorts unless @sorts.blank?
+      b
+    end
+
+  end
+end

data/lib/elastic_queue/queueable.rb

@@ -0,0 +1,66 @@
+module ElasticQueue
+  module Queueable
+    extend ActiveSupport::Concern
+
+    included do
+      after_commit :index_for_queues, if: :persisted?
+      after_touch :index_for_queues, if: :persisted?
+      before_destroy :remove_from_queue_indices
+    end
+
+    module ClassMethods
+
+      def queues(*queues)
+        @queues ||= queues
+      end
+
+      def queue_classes
+        queues.map { |q| q.to_s.camelize.constantize }
+      end
+
+      def queue_attributes(*attributes)
+        @queue_attributes ||= attributes
+      end
+
+      alias_method :analyzed_queue_attributes, :queue_attributes
+
+      def not_analyzed_queue_attributes(*attributes)
+        @not_analyzed_queue_attributes ||= attributes
+      end
+
+      # the union of analyzed and not_analyzed attributes
+      def all_queue_attributes
+        @queue_attributes.to_a | @not_analyzed_queue_attributes.to_a
+      end
+
+      def queue_mapping
+        return if @not_analyzed_queue_attributes.blank?
+        properties = {}
+        @not_analyzed_queue_attributes.each do |a|
+          properties[a.to_sym] = { type: :string, index: :not_analyzed }
+        end
+        { self.to_s.underscore.to_sym => { properties: properties } }
+      end
+
+    end
+
+    def indexed_for_queue
+      index = { id: id, model: self.class.to_s.underscore }
+      self.class.all_queue_attributes.each do |attr|
+        val = send(attr)
+        val = val.to_s(:db) if val.is_a? Date
+        index[attr] = val
+      end
+      index
+    end
+
+    def index_for_queues
+      self.class.queue_classes.each { |q| q.send(:upsert_model, self) }
+    end
+
+    def remove_from_queue_indices
+      self.class.queue_classes.each { |q| q.send(:remove_model, self) }
+    end
+
+  end
+end

data/lib/elastic_queue/results.rb

@@ -0,0 +1,67 @@
+require 'will_paginate/collection'
+
+module ElasticQueue
+
+  class Results
+
+    attr_reader :results
+
+    def initialize(queue, search_results, query_options)
+      @queue = queue
+      @instantiated_queue_items = instantiate_queue_items(search_results)
+      @start = query_options.page
+      @per_page = query_options.per_page
+      @total = search_results[:hits][:total]
+      @results = WillPaginate::Collection.create(@start, @per_page, @total) do |pager|
+        pager.replace(@instantiated_queue_items)
+      end
+    end
+
+    def paginate
+      @results
+    end
+
+    def instantiate_queue_items(search_results)
+      grouped_results, sort_order = group_sorted_results(search_results)
+      records = fetch_records(grouped_results)
+      sort_records(records, sort_order)
+    end
+
+    private
+
+    # group the results by { class_name: [ids] } and save their sorted order
+    def group_sorted_results(search_results)
+      grouped_results = {}
+      sort_order = {}
+      search_results[:hits][:hits].each_with_index do |result, index|
+        model = result[:_source][:model].to_sym
+        model_id = result[:_source][:id]
+        sort_order["#{model}_#{model_id}"] = index # save the sort order
+        grouped_results[model] ||= []
+        grouped_results[model] << model_id
+      end
+      [grouped_results, sort_order]
+    end
+
+    # take a hash of { model_name: [ids] } and return a list of records
+    def fetch_records(grouped_results)
+      records = []
+      grouped_results.each do |model, ids|
+        klass = model.to_s.camelize.constantize
+        if @queue.eager_loads && @queue.eager_loads[model]
+          records += klass.includes(@queue.eager_loads[model]).find_all_by_id(ids)
+        else
+          records += klass.find_all_by_id(ids)
+        end
+      end
+      records
+    end
+
+    def sort_records(records, sort_order)
+      records.sort do |a, b|
+        sort_order["#{a.class.name.underscore}_#{a.id}"] <=> sort_order["#{b.class.name.underscore}_#{b.id}"]
+      end
+    end
+
+  end
+end

data/lib/elastic_queue/search.rb

@@ -0,0 +1,5 @@
+module ElasticQueue
+  class Search
+    # TODO
+  end
+end

data/lib/elastic_queue/sorts.rb

@@ -0,0 +1,23 @@
+module ElasticQueue
+  module Sorts
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+    end
+
+    def options_to_sorts(options)
+      options.map { |k, v| option_to_sort(k, v) }
+    end
+
+    private
+
+    def option_to_sort(key, value)
+      single_sort(key, value)
+    end
+
+    def single_sort(order_by, order)
+      { order_by => { order: order, ignore_unmapped: true } }
+    end
+
+  end
+end

data/lib/elastic_queue/version.rb

@@ -0,0 +1,3 @@
+module ElasticQueue
+  VERSION = '0.0.2'
+end