elastic_queue 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.rubocop.yml +20 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +56 -0
- data/LICENSE +20 -0
- data/README.md +12 -0
- data/Rakefile +25 -0
- data/elastic_queue.gemspec +28 -0
- data/lib/elastic_queue/base.rb +52 -0
- data/lib/elastic_queue/filters.rb +47 -0
- data/lib/elastic_queue/percolation.rb +42 -0
- data/lib/elastic_queue/persistence.rb +68 -0
- data/lib/elastic_queue/query.rb +62 -0
- data/lib/elastic_queue/query_options.rb +48 -0
- data/lib/elastic_queue/queueable.rb +66 -0
- data/lib/elastic_queue/results.rb +67 -0
- data/lib/elastic_queue/search.rb +5 -0
- data/lib/elastic_queue/sorts.rb +23 -0
- data/lib/elastic_queue/version.rb +3 -0
- data/lib/elastic_queue.rb +7 -0
- data/spec/factories.rb +64 -0
- data/spec/lib/elastic_queue_functional_spec.rb +276 -0
- data/spec/lib/elastic_queue_unit_spec.rb +220 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/support/elastic_queue_helper.rb +18 -0
- metadata +175 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5fba6995c9fb76fcb3b2f7e0e204bf5c955e243b
|
4
|
+
data.tar.gz: a47b1f04bf521b2a4c42f1fee5f2be4feee51256
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f7e36b7ac21298dd00e897b1f7c4e16f02a0b20c3bd21b33d4e68fded773acb9ebd712dba768bd73ea216f2f3b52ccc85989168997c27f57dd821f6b8044866e
|
7
|
+
data.tar.gz: 16409af91a7ae6020d5844a68b4ceddfbe0a53283d1bb3e7cb066210d400c40e0963ba7082a0ecaa17700a46cade6ecaa404ee585f6063dd39675c8895b23b1c
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# This is the configuration rubocop, a ruby source code linter and checker.
|
2
|
+
Documentation:
|
3
|
+
Description: 'Document classes and non-namespace modules.'
|
4
|
+
Enabled: false
|
5
|
+
|
6
|
+
FinalNewline:
|
7
|
+
Description: 'Checks for a final newline in a source file.'
|
8
|
+
Enabled: false
|
9
|
+
|
10
|
+
LineLength:
|
11
|
+
Description: 'Limit lines to 79 characters.'
|
12
|
+
Enabled: false
|
13
|
+
|
14
|
+
MethodLength:
|
15
|
+
Description: 'Avoid methods longer than 10 lines of code.'
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
RescueModifier:
|
19
|
+
Description: 'Avoid using rescue in its modifier form.'
|
20
|
+
Enabled: false
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
elastic-queue (0.0.2)
|
5
|
+
activesupport
|
6
|
+
elasticsearch
|
7
|
+
|
8
|
+
GEM
|
9
|
+
remote: https://rubygems.org/
|
10
|
+
specs:
|
11
|
+
activesupport (4.0.1)
|
12
|
+
i18n (~> 0.6, >= 0.6.4)
|
13
|
+
minitest (~> 4.2)
|
14
|
+
multi_json (~> 1.3)
|
15
|
+
thread_safe (~> 0.1)
|
16
|
+
tzinfo (~> 0.3.37)
|
17
|
+
atomic (1.1.14)
|
18
|
+
diff-lcs (1.2.5)
|
19
|
+
elasticsearch (0.4.1)
|
20
|
+
elasticsearch-api (= 0.4.1)
|
21
|
+
elasticsearch-transport (= 0.4.1)
|
22
|
+
elasticsearch-api (0.4.1)
|
23
|
+
multi_json
|
24
|
+
elasticsearch-transport (0.4.1)
|
25
|
+
faraday
|
26
|
+
multi_json
|
27
|
+
factory_girl (4.2.0)
|
28
|
+
activesupport (>= 3.0.0)
|
29
|
+
faraday (0.8.8)
|
30
|
+
multipart-post (~> 1.2.0)
|
31
|
+
i18n (0.6.9)
|
32
|
+
minitest (4.7.5)
|
33
|
+
multi_json (1.8.2)
|
34
|
+
multipart-post (1.2.0)
|
35
|
+
rake (10.1.0)
|
36
|
+
rspec (2.14.1)
|
37
|
+
rspec-core (~> 2.14.0)
|
38
|
+
rspec-expectations (~> 2.14.0)
|
39
|
+
rspec-mocks (~> 2.14.0)
|
40
|
+
rspec-core (2.14.7)
|
41
|
+
rspec-expectations (2.14.4)
|
42
|
+
diff-lcs (>= 1.1.3, < 2.0)
|
43
|
+
rspec-mocks (2.14.4)
|
44
|
+
thread_safe (0.1.3)
|
45
|
+
atomic
|
46
|
+
tzinfo (0.3.38)
|
47
|
+
|
48
|
+
PLATFORMS
|
49
|
+
ruby
|
50
|
+
|
51
|
+
DEPENDENCIES
|
52
|
+
bundler (>= 1.0.0)
|
53
|
+
elastic-queue!
|
54
|
+
factory_girl
|
55
|
+
rake
|
56
|
+
rspec (~> 2.6)
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2014 RuthThompson
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# ElasticQueue
|
2
|
+
|
3
|
+
[![Code Climate](https://codeclimate.com/github/RuthThompson/elastic_queue.png)](https://codeclimate.com/github/RuthThompson/elastic_queue)
|
4
|
+
|
5
|
+
A queueing system built on top of elasticsearch.
|
6
|
+
|
7
|
+
### Usage
|
8
|
+
TODO
|
9
|
+
|
10
|
+
### How it Works
|
11
|
+
|
12
|
+
TODO
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
require 'rake'
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
Bundler::GemHelper.install_tasks
|
6
|
+
|
7
|
+
desc 'Default: run the specs and features.'
|
8
|
+
task :default do
|
9
|
+
system('bundle exec rake -s appraisal spec:unit spec:acceptance features;')
|
10
|
+
end
|
11
|
+
|
12
|
+
namespace :spec do
|
13
|
+
desc 'Run unit specs'
|
14
|
+
RSpec::Core::RakeTask.new('unit') do |t|
|
15
|
+
t.pattern = 'spec/{*_spec.rb,factory_girl/**/*_spec.rb}'
|
16
|
+
end
|
17
|
+
|
18
|
+
desc 'Run acceptance specs'
|
19
|
+
RSpec::Core::RakeTask.new('acceptance') do |t|
|
20
|
+
t.pattern = 'spec/acceptance/**/*_spec.rb'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
desc 'Run the unit and acceptance specs'
|
25
|
+
task :spec => ['spec:unit', 'spec:acceptance']
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require './lib/elastic_queue/version'
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'elastic_queue'
|
5
|
+
s.version = ElasticQueue::VERSION
|
6
|
+
s.platform = Gem::Platform::RUBY
|
7
|
+
s.summary = 'A queueing system built on top of elasticsearch.'
|
8
|
+
s.description = 'A library for storing and filtering documents on elastic search with a queue paradigm for retrieval.'
|
9
|
+
s.license = 'MIT'
|
10
|
+
|
11
|
+
s.required_ruby_version = '>= 1.9.2'
|
12
|
+
|
13
|
+
s.authors = ['Ruth Thompson', 'Rob Law']
|
14
|
+
s.email = %w[ ruth@flywheelnetworks.com rob@flywheelnetworks.com ]
|
15
|
+
s.homepage = 'https://github.com/RuthThompson/elastic_queue'
|
16
|
+
|
17
|
+
s.require_paths = %w[ lib ]
|
18
|
+
s.files = `git ls-files`.split("\n")
|
19
|
+
s.test_files = Dir['spec/**/*.rb']
|
20
|
+
|
21
|
+
s.add_dependency 'activesupport'
|
22
|
+
s.add_dependency 'elasticsearch'
|
23
|
+
s.add_dependency 'will_paginate'
|
24
|
+
s.add_development_dependency 'bundler', '>= 1.0.0'
|
25
|
+
s.add_development_dependency 'rspec', '~> 2.6'
|
26
|
+
s.add_development_dependency 'factory_girl'
|
27
|
+
s.add_development_dependency 'rake'
|
28
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'elasticsearch'
|
2
|
+
require 'elastic_queue/persistence'
|
3
|
+
require 'elastic_queue/query'
|
4
|
+
|
5
|
+
module ElasticQueue
|
6
|
+
class Base
|
7
|
+
include Persistence
|
8
|
+
# include Percolation
|
9
|
+
|
10
|
+
def self.search_client
|
11
|
+
Elasticsearch::Client.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.models(*models)
|
15
|
+
@models = models
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.model_names
|
19
|
+
raise NotImplementedError, "No models defined in #{self.class}" unless defined?(@models)
|
20
|
+
@models
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.model_classes
|
24
|
+
model_names.map { |s| s.to_s.camelize.constantize }
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.index_name
|
28
|
+
@index_name ||= to_s.underscore
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.eager_load(includes)
|
32
|
+
@eager_loads = includes
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.eager_loads
|
36
|
+
@eager_loads
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.query(options = {})
|
40
|
+
Query.new(self, options)
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.filter(options)
|
44
|
+
query.filter(options)
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.count
|
48
|
+
query.count
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module ElasticQueue
|
2
|
+
module Filters
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
module ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
def options_to_filters(options)
|
9
|
+
options.map { |k, v| option_to_filter(k, v) }.flatten
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def option_to_filter(key, value)
|
15
|
+
if value.is_a? Array
|
16
|
+
or_filter(key, value)
|
17
|
+
elsif value.is_a? Hash
|
18
|
+
# date?
|
19
|
+
time_filter(key, value)
|
20
|
+
else
|
21
|
+
term_filter(key, value)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def or_filter(term, values)
|
26
|
+
{ or: values.map { |v| term_filter(term, v) } }
|
27
|
+
end
|
28
|
+
|
29
|
+
def term_filter(term, value)
|
30
|
+
{ term: { term => value } }
|
31
|
+
end
|
32
|
+
|
33
|
+
# take something like follow_up: { before: 'hii', after: 'low' }
|
34
|
+
def time_filter(term, value)
|
35
|
+
value.map do |k, v|
|
36
|
+
comparator = k.to_sym.in?([:after, :greater_than, :gt]) ? :gt : :lt
|
37
|
+
range_filter(term, v, comparator)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# like term filter but for comparison queries
|
42
|
+
def range_filter(term, value, comparator)
|
43
|
+
{ range: { term => { comparator => value } } }
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module ElasticQueue
|
2
|
+
module Percolation
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
module ClassMethods
|
6
|
+
|
7
|
+
# def percolator_queries
|
8
|
+
# queries = {}
|
9
|
+
# query = { 'query' => {'match_all' => {}} }
|
10
|
+
# search = search_client.search index: '_percolator', body: query.to_json, size: 1000
|
11
|
+
# debugger
|
12
|
+
# search['hits']['hits'].each { |hit| queries[hit['_id']] = hit['_source'] }
|
13
|
+
# queries
|
14
|
+
# end
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
# def reverse_search(instance)
|
19
|
+
# body = { 'doc' => instance.indexed_for_queue }
|
20
|
+
# search_client.percolate index: index_name, body: body.to_json
|
21
|
+
# end
|
22
|
+
|
23
|
+
# def register_percolator_query(name, opts)
|
24
|
+
# search_client.index index: '_percolator', type: index_name, id: name, body: translate_opts_to_query_for_percolator(opts)
|
25
|
+
# end
|
26
|
+
|
27
|
+
# def unregister_percolator_query(name)
|
28
|
+
# SEARCH_CLIENT.delete index: '_percolator', type: index_name, id: name
|
29
|
+
# end
|
30
|
+
|
31
|
+
# def model_in_queue?(model, queue_opts)
|
32
|
+
# return false unless model_names.include?(model.class.to_s.underscore.to_sym)
|
33
|
+
# search_id = SecureRandom.uuid
|
34
|
+
# body = { 'doc' => model.indexed_for_queue, 'query' => {'term' => {'_id' => search_id}} }
|
35
|
+
# SEARCH_CLIENT.index index: '_percolator', type: 'dynamic_percolator', id: search_id, body: translate_opts_to_query_for_percolator(queue_opts), refresh: true
|
36
|
+
# search = SEARCH_CLIENT.percolate index: 'dynamic_percolator', body: body.to_json
|
37
|
+
# SEARCH_CLIENT.delete index: '_percolator', type: 'dynamic_percolator', id: search_id
|
38
|
+
# search['matches'].length == 1
|
39
|
+
# end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# TODO: might want to move these to an Index class
|
2
|
+
module ElasticQueue
|
3
|
+
module Persistence
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
module ClassMethods
|
6
|
+
|
7
|
+
def index_exists?
|
8
|
+
search_client.indices.exists index: index_name
|
9
|
+
end
|
10
|
+
|
11
|
+
def reset_index
|
12
|
+
delete_index if index_exists?
|
13
|
+
create_index
|
14
|
+
end
|
15
|
+
|
16
|
+
def create_index
|
17
|
+
search_client.indices.create index: index_name
|
18
|
+
add_mappings
|
19
|
+
end
|
20
|
+
|
21
|
+
def delete_index
|
22
|
+
search_client.indices.delete index: index_name
|
23
|
+
end
|
24
|
+
|
25
|
+
# not using it, but it is nice for debugging
|
26
|
+
def refresh_index
|
27
|
+
search_client.indices.refresh index: index_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def bulk_index(batch_size = 10_000)
|
31
|
+
create_index unless index_exists?
|
32
|
+
model_classes.each do |klass|
|
33
|
+
# modelclass(model).includes(associations_for_index(model)).
|
34
|
+
index_type = klass.to_s.underscore
|
35
|
+
klass.find_in_batches(batch_size: batch_size) do |batch|
|
36
|
+
body = []
|
37
|
+
batch.each do |instance|
|
38
|
+
body << { index: { _index: index_name, _id: instance.id, _type: index_type, data: instance.indexed_for_queue } }
|
39
|
+
end
|
40
|
+
search_client.bulk body: body
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def add_mappings
|
46
|
+
model_classes.each do |klass|
|
47
|
+
search_client.indices.put_mapping index: index_name, type: klass.to_s.underscore, body: klass.queue_mapping
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# TODO: move these to an instance?
|
52
|
+
def index_model(instance)
|
53
|
+
search_client.index index: index_name, id: instance.id, type: instance.class.to_s.underscore, body: instance.indexed_for_queue
|
54
|
+
end
|
55
|
+
|
56
|
+
def upsert_model(instance)
|
57
|
+
body = { doc: instance.indexed_for_queue, doc_as_upsert: true }
|
58
|
+
search_client.update index: index_name, id: instance.id, type: instance.class.to_s.underscore, body: body, refresh: true
|
59
|
+
end
|
60
|
+
|
61
|
+
def remove_model(instance)
|
62
|
+
search_client.delete index: index_name, id: instance.id, type: instance.class.to_s.underscore
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'elastic_queue/query_options'
|
2
|
+
require 'elastic_queue/results'
|
3
|
+
|
4
|
+
module ElasticQueue
|
5
|
+
class Query
|
6
|
+
|
7
|
+
def initialize(queue, options = {})
|
8
|
+
@queue = queue
|
9
|
+
@options = QueryOptions.new(options)
|
10
|
+
end
|
11
|
+
|
12
|
+
def filter(options)
|
13
|
+
@options.add_filter(options)
|
14
|
+
self
|
15
|
+
end
|
16
|
+
|
17
|
+
def filters
|
18
|
+
@options.filters
|
19
|
+
end
|
20
|
+
|
21
|
+
def sort(options)
|
22
|
+
@options.add_sort(options)
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
def sorts
|
27
|
+
@options.sorts
|
28
|
+
end
|
29
|
+
|
30
|
+
def paginate(options = {})
|
31
|
+
options.each { |k, v| @options.send("#{k}=", v) }
|
32
|
+
all.paginate
|
33
|
+
end
|
34
|
+
|
35
|
+
def all
|
36
|
+
@results ||= Results.new(@queue, execute, @options)
|
37
|
+
end
|
38
|
+
|
39
|
+
def count
|
40
|
+
res = execute(count: true)
|
41
|
+
res[:hits][:total].to_i
|
42
|
+
end
|
43
|
+
|
44
|
+
def execute(count: false)
|
45
|
+
search_type = count ? 'count' : 'query_then_fetch'
|
46
|
+
begin
|
47
|
+
search = @queue.search_client.search index: @queue.index_name, body: @options.body, search_type: search_type, from: @options.from, size: @options.per_page
|
48
|
+
# search[:page] = @page
|
49
|
+
# search = substitute_page(opts, search) if !count && opts[:page_substitution_ok] && search['hits']['hits'].length == 0 && search['hits']['total'] != 0
|
50
|
+
rescue Elasticsearch::Transport::Transport::Errors::BadRequest
|
51
|
+
search = failed_search
|
52
|
+
end
|
53
|
+
search.with_indifferent_access
|
54
|
+
end
|
55
|
+
|
56
|
+
def failed_search
|
57
|
+
{ page: 0, hits: { hits: [], total: 0 } }
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'elastic_queue/filters'
|
2
|
+
require 'elastic_queue/sorts'
|
3
|
+
|
4
|
+
module ElasticQueue
|
5
|
+
class QueryOptions
|
6
|
+
include Filters
|
7
|
+
include Sorts
|
8
|
+
|
9
|
+
attr_reader :filters, :sorts
|
10
|
+
attr_accessor :per_page
|
11
|
+
|
12
|
+
def initialize(options = {})
|
13
|
+
@options = { per_page: 30, page: 1 }.merge(options)
|
14
|
+
@filters = { and: [] }.with_indifferent_access
|
15
|
+
@sorts = []
|
16
|
+
self.per_page = @options[:per_page]
|
17
|
+
self.page = @options[:page]
|
18
|
+
end
|
19
|
+
|
20
|
+
def add_filter(options)
|
21
|
+
@filters[:and] += options_to_filters(options)
|
22
|
+
end
|
23
|
+
|
24
|
+
def add_sort(options)
|
25
|
+
@sorts += options_to_sorts(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def from
|
29
|
+
(page - 1) * per_page
|
30
|
+
end
|
31
|
+
|
32
|
+
def page=(num)
|
33
|
+
@page = num.to_i unless num.blank?
|
34
|
+
end
|
35
|
+
|
36
|
+
def page
|
37
|
+
@page
|
38
|
+
end
|
39
|
+
|
40
|
+
def body
|
41
|
+
b = {}
|
42
|
+
b[:filter] = @filters unless @filters[:and].blank?
|
43
|
+
b[:sort] = @sorts unless @sorts.blank?
|
44
|
+
b
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
module ElasticQueue
|
2
|
+
module Queueable
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
included do
|
6
|
+
after_commit :index_for_queues, if: :persisted?
|
7
|
+
after_touch :index_for_queues, if: :persisted?
|
8
|
+
before_destroy :remove_from_queue_indices
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
|
13
|
+
def queues(*queues)
|
14
|
+
@queues ||= queues
|
15
|
+
end
|
16
|
+
|
17
|
+
def queue_classes
|
18
|
+
queues.map { |q| q.to_s.camelize.constantize }
|
19
|
+
end
|
20
|
+
|
21
|
+
def queue_attributes(*attributes)
|
22
|
+
@queue_attributes ||= attributes
|
23
|
+
end
|
24
|
+
|
25
|
+
alias_method :analyzed_queue_attributes, :queue_attributes
|
26
|
+
|
27
|
+
def not_analyzed_queue_attributes(*attributes)
|
28
|
+
@not_analyzed_queue_attributes ||= attributes
|
29
|
+
end
|
30
|
+
|
31
|
+
# the union of analyzed and not_analyzed attributes
|
32
|
+
def all_queue_attributes
|
33
|
+
@queue_attributes.to_a | @not_analyzed_queue_attributes.to_a
|
34
|
+
end
|
35
|
+
|
36
|
+
def queue_mapping
|
37
|
+
return if @not_analyzed_queue_attributes.blank?
|
38
|
+
properties = {}
|
39
|
+
@not_analyzed_queue_attributes.each do |a|
|
40
|
+
properties[a.to_sym] = { type: :string, index: :not_analyzed }
|
41
|
+
end
|
42
|
+
{ self.to_s.underscore.to_sym => { properties: properties } }
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
def indexed_for_queue
|
48
|
+
index = { id: id, model: self.class.to_s.underscore }
|
49
|
+
self.class.all_queue_attributes.each do |attr|
|
50
|
+
val = send(attr)
|
51
|
+
val = val.to_s(:db) if val.is_a? Date
|
52
|
+
index[attr] = val
|
53
|
+
end
|
54
|
+
index
|
55
|
+
end
|
56
|
+
|
57
|
+
def index_for_queues
|
58
|
+
self.class.queue_classes.each { |q| q.send(:upsert_model, self) }
|
59
|
+
end
|
60
|
+
|
61
|
+
def remove_from_queue_indices
|
62
|
+
self.class.queue_classes.each { |q| q.send(:remove_model, self) }
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'will_paginate/collection'
|
2
|
+
|
3
|
+
module ElasticQueue
|
4
|
+
|
5
|
+
class Results
|
6
|
+
|
7
|
+
attr_reader :results
|
8
|
+
|
9
|
+
def initialize(queue, search_results, query_options)
|
10
|
+
@queue = queue
|
11
|
+
@instantiated_queue_items = instantiate_queue_items(search_results)
|
12
|
+
@start = query_options.page
|
13
|
+
@per_page = query_options.per_page
|
14
|
+
@total = search_results[:hits][:total]
|
15
|
+
@results = WillPaginate::Collection.create(@start, @per_page, @total) do |pager|
|
16
|
+
pager.replace(@instantiated_queue_items)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def paginate
|
21
|
+
@results
|
22
|
+
end
|
23
|
+
|
24
|
+
def instantiate_queue_items(search_results)
|
25
|
+
grouped_results, sort_order = group_sorted_results(search_results)
|
26
|
+
records = fetch_records(grouped_results)
|
27
|
+
sort_records(records, sort_order)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
# group the results by { class_name: [ids] } and save their sorted order
|
33
|
+
def group_sorted_results(search_results)
|
34
|
+
grouped_results = {}
|
35
|
+
sort_order = {}
|
36
|
+
search_results[:hits][:hits].each_with_index do |result, index|
|
37
|
+
model = result[:_source][:model].to_sym
|
38
|
+
model_id = result[:_source][:id]
|
39
|
+
sort_order["#{model}_#{model_id}"] = index # save the sort order
|
40
|
+
grouped_results[model] ||= []
|
41
|
+
grouped_results[model] << model_id
|
42
|
+
end
|
43
|
+
[grouped_results, sort_order]
|
44
|
+
end
|
45
|
+
|
46
|
+
# take a hash of { model_name: [ids] } and return a list of records
|
47
|
+
def fetch_records(grouped_results)
|
48
|
+
records = []
|
49
|
+
grouped_results.each do |model, ids|
|
50
|
+
klass = model.to_s.camelize.constantize
|
51
|
+
if @queue.eager_loads && @queue.eager_loads[model]
|
52
|
+
records += klass.includes(@queue.eager_loads[model]).find_all_by_id(ids)
|
53
|
+
else
|
54
|
+
records += klass.find_all_by_id(ids)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
records
|
58
|
+
end
|
59
|
+
|
60
|
+
def sort_records(records, sort_order)
|
61
|
+
records.sort do |a, b|
|
62
|
+
sort_order["#{a.class.name.underscore}_#{a.id}"] <=> sort_order["#{b.class.name.underscore}_#{b.id}"]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module ElasticQueue
|
2
|
+
module Sorts
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
module ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
def options_to_sorts(options)
|
9
|
+
options.map { |k, v| option_to_sort(k, v) }
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def option_to_sort(key, value)
|
15
|
+
single_sort(key, value)
|
16
|
+
end
|
17
|
+
|
18
|
+
def single_sort(order_by, order)
|
19
|
+
{ order_by => { order: order, ignore_unmapped: true } }
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|