es-elasticity 0.2.11 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +5 -1
- data/elasticity.gemspec +1 -0
- data/lib/elasticity.rb +33 -5
- data/lib/elasticity/bulk.rb +53 -0
- data/lib/elasticity/config.rb +28 -0
- data/lib/elasticity/document.rb +89 -39
- data/lib/elasticity/instrumented_client.rb +35 -0
- data/lib/elasticity/log_subscriber.rb +49 -0
- data/lib/elasticity/multi_search.rb +22 -15
- data/lib/elasticity/railtie.rb +3 -17
- data/lib/elasticity/search.rb +190 -123
- data/lib/elasticity/strategies.rb +15 -0
- data/lib/elasticity/strategies/alias_index.rb +255 -0
- data/lib/elasticity/strategies/single_index.rb +97 -0
- data/lib/elasticity/version.rb +1 -1
- data/spec/functional/persistence_spec.rb +167 -0
- data/spec/rspec_config.rb +7 -5
- data/spec/units/document_spec.rb +25 -34
- data/spec/units/multi_search_spec.rb +11 -3
- data/spec/units/search_spec.rb +41 -31
- data/spec/units/{index_spec.rb → strategies/single_index_spec.rb} +7 -10
- metadata +27 -6
- data/lib/elasticity/index.rb +0 -118
- data/lib/elasticity_base.rb +0 -56
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c7bc6c25943feeda4da146553b189c61bbdad216
|
4
|
+
data.tar.gz: 9f07d0aa831e187b094481e96178fce66c923139
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d9109393c9161297e7a103d821f0a91599b8dae175174b228380f82d3db975a9d3d32902cc5ea671da301bc90f8d4705db4ed232ef019d0427c82bf49f032dd
|
7
|
+
data.tar.gz: bc7fb37f2ded19d2dd98e636a2f346aa078b3bee37c4e49ff3d46278b3d6e1f1d0407e385f04f344887b9bafd6f2b6c8f667fc983f972c5e6ab6725719b4c7be
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/elasticity.gemspec
CHANGED
@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.add_development_dependency "oj"
|
27
27
|
spec.add_development_dependency "pry"
|
28
28
|
spec.add_development_dependency "codeclimate-test-reporter"
|
29
|
+
spec.add_development_dependency "redis"
|
29
30
|
|
30
31
|
spec.add_dependency "activesupport", "~> 4.0"
|
31
32
|
spec.add_dependency "activemodel", "~> 4.0"
|
data/lib/elasticity.rb
CHANGED
@@ -1,5 +1,33 @@
|
|
1
|
-
require "
|
2
|
-
require "
|
3
|
-
|
4
|
-
|
5
|
-
require "
|
1
|
+
require "rubygems"
|
2
|
+
require "bundler/setup"
|
3
|
+
Bundler.setup
|
4
|
+
|
5
|
+
require "active_support"
|
6
|
+
require "active_support/core_ext"
|
7
|
+
require "active_model"
|
8
|
+
require "elasticsearch"
|
9
|
+
|
10
|
+
module Elasticity
|
11
|
+
autoload :Bulk, "elasticity/bulk"
|
12
|
+
autoload :Config, "elasticity/config"
|
13
|
+
autoload :Document, "elasticity/document"
|
14
|
+
autoload :InstrumentedClient, "elasticity/instrumented_client"
|
15
|
+
autoload :LogSubscriber, "elasticity/log_subscriber"
|
16
|
+
autoload :MultiSearch, "elasticity/multi_search"
|
17
|
+
autoload :Search, "elasticity/search"
|
18
|
+
autoload :Strategies, "elasticity/strategies"
|
19
|
+
|
20
|
+
def self.configure
|
21
|
+
@config = Config.new
|
22
|
+
yield(@config)
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.config
|
26
|
+
return @config if defined?(@config)
|
27
|
+
@config = Config.new
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
if defined?(Rails)
|
32
|
+
require "elasticity/railtie"
|
33
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Elasticity
|
2
|
+
class Bulk
|
3
|
+
def initialize(client)
|
4
|
+
@client = client
|
5
|
+
@operations = []
|
6
|
+
end
|
7
|
+
|
8
|
+
def index(index_name, type, id, attributes)
|
9
|
+
@operations << { index: { _index: index_name, _type: type, _id: id, data: attributes }}
|
10
|
+
end
|
11
|
+
|
12
|
+
def delete(index_name, type, id)
|
13
|
+
@operations << { delete: { _index: index_name, _type: type, _id: id }}
|
14
|
+
end
|
15
|
+
|
16
|
+
def execute
|
17
|
+
@client.bulk(body: @operations)
|
18
|
+
end
|
19
|
+
|
20
|
+
class Index < Bulk
|
21
|
+
def initialize(client, index_name)
|
22
|
+
super(client)
|
23
|
+
@index_name = index_name
|
24
|
+
end
|
25
|
+
|
26
|
+
def index(type, id, attributes)
|
27
|
+
super(@index_name, type, id, attributes)
|
28
|
+
end
|
29
|
+
|
30
|
+
def delete(type, id)
|
31
|
+
super(@index_name, type, id)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class Alias < Bulk
|
36
|
+
def initialize(client, update_alias, delete_indexes)
|
37
|
+
super(client)
|
38
|
+
@update_alias = update_alias
|
39
|
+
@delete_indexes = delete_indexes
|
40
|
+
end
|
41
|
+
|
42
|
+
def index(type, id, attributes)
|
43
|
+
super(@update_alias, type, id, attributes)
|
44
|
+
end
|
45
|
+
|
46
|
+
def delete(type, id)
|
47
|
+
@delete_indexes.each do |index|
|
48
|
+
super(index, type, id)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Elasticity
|
2
|
+
class Config
|
3
|
+
def client=(client)
|
4
|
+
@client = Elasticity::InstrumentedClient.new(client)
|
5
|
+
end
|
6
|
+
|
7
|
+
def client
|
8
|
+
return @client if defined?(@client)
|
9
|
+
self.client = Elasticsearch::Client.new
|
10
|
+
@client
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_writer :settings, :namespace, :pretty_json
|
14
|
+
|
15
|
+
def settings
|
16
|
+
return @settings if defined?(@settings)
|
17
|
+
@settings = {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def namespace
|
21
|
+
@namespace
|
22
|
+
end
|
23
|
+
|
24
|
+
def pretty_json
|
25
|
+
@pretty_json || false
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/elasticity/document.rb
CHANGED
@@ -2,89 +2,131 @@ module Elasticity
|
|
2
2
|
class Document
|
3
3
|
include ::ActiveModel::Model
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
class NotConfigured < StandardError; end
|
6
|
+
|
7
|
+
Config = Struct.new(:index_base_name, :document_type, :mapping, :strategy)
|
8
|
+
|
9
|
+
# Configure the given klass, changing default parameters and resetting
|
10
|
+
# some of the internal state.
|
11
|
+
def self.configure
|
12
|
+
@config = Config.new
|
13
|
+
@config.strategy = Strategies::SingleIndex
|
14
|
+
yield(@config)
|
15
|
+
end
|
16
|
+
|
17
|
+
# Returns the stategy class being used.
|
18
|
+
# Check Elasticity::Strategies for more information.
|
19
|
+
def self.strategy
|
20
|
+
if @config.nil? || @config.strategy.nil?
|
21
|
+
raise NotConfigured, "#{self} has not been configured, make sure you call the configure method"
|
22
|
+
end
|
23
|
+
|
24
|
+
return @strategy if defined?(@strategy)
|
25
|
+
|
26
|
+
if namespace = Elasticity.config.namespace
|
27
|
+
index_base_name = "#{namespace}_#{@config.index_base_name}"
|
28
|
+
end
|
29
|
+
|
30
|
+
@strategy = @config.strategy.new(Elasticity.config.client, index_base_name)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Document type
|
34
|
+
def self.document_type
|
35
|
+
if @config.nil? || @config.document_type.blank?
|
36
|
+
raise NotConfigured, "#{self} has not been configured, make sure you call the configure method"
|
37
|
+
end
|
38
|
+
|
39
|
+
@config.document_type
|
40
|
+
end
|
41
|
+
|
42
|
+
# Document type
|
43
|
+
def self.mapping
|
44
|
+
if @config.nil? || @config.mapping.blank?
|
45
|
+
raise NotConfigured, "#{self} has not been configured, make sure you call the configure method"
|
46
|
+
end
|
47
|
+
|
48
|
+
@config.mapping
|
9
49
|
end
|
10
50
|
|
11
51
|
# Creates the index for this document
|
12
52
|
def self.create_index
|
13
|
-
self.
|
53
|
+
self.strategy.create_if_undefined(settings: Elasticity.config.settings, mappings: { document_type => mapping })
|
14
54
|
end
|
15
55
|
|
16
56
|
# Re-creates the index for this document
|
17
57
|
def self.recreate_index
|
18
|
-
self.
|
58
|
+
self.strategy.recreate(settings: Elasticity.config.settings, mappings: { document_type => mapping })
|
19
59
|
end
|
20
60
|
|
21
61
|
# Deletes the index
|
22
62
|
def self.delete_index
|
23
|
-
self.
|
63
|
+
self.strategy.delete
|
24
64
|
end
|
25
65
|
|
26
|
-
#
|
27
|
-
def self.
|
28
|
-
|
29
|
-
@index = nil
|
66
|
+
# Does the index exist?
|
67
|
+
def self.index_exists?
|
68
|
+
!self.strategy.missing?
|
30
69
|
end
|
31
70
|
|
32
|
-
#
|
33
|
-
def self.
|
34
|
-
|
35
|
-
|
36
|
-
if namespace = Elasticity.config.namespace
|
37
|
-
name = "#{namespace}_#{name}"
|
38
|
-
end
|
39
|
-
|
40
|
-
name
|
71
|
+
# Remap
|
72
|
+
def self.remap!
|
73
|
+
self.strategy.remap(settings: Elasticity.config.settings, mappings: { document_type => mapping })
|
41
74
|
end
|
42
75
|
|
43
|
-
#
|
44
|
-
def self.
|
45
|
-
|
46
|
-
@document_type = self.name.demodulize.underscore
|
76
|
+
# Flushes the index, forcing any writes
|
77
|
+
def self.flush_index
|
78
|
+
self.strategy.flush
|
47
79
|
end
|
48
80
|
|
49
|
-
#
|
50
|
-
def self.
|
51
|
-
|
52
|
-
|
81
|
+
# Creates a instance of a document from a ElasticSearch hit data.
|
82
|
+
def self.from_hit(hit_data)
|
83
|
+
attrs = hit_data["_source"].merge(_id: hit_data['_id'])
|
84
|
+
|
85
|
+
if hit_data["highlight"]
|
86
|
+
highlighted_attrs = attrs.dup
|
87
|
+
attrs_set = Set.new
|
53
88
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
89
|
+
hit_data["highlight"].each do |name, v|
|
90
|
+
name = name.gsub(/\..*\z/, '')
|
91
|
+
next if attrs_set.include?(name)
|
92
|
+
highlighted_attrs[name] = v
|
93
|
+
attrs_set << name
|
94
|
+
end
|
95
|
+
|
96
|
+
highlighted = new(highlighted_attrs)
|
97
|
+
end
|
98
|
+
|
99
|
+
new(attrs.merge(highlighted: highlighted))
|
59
100
|
end
|
60
101
|
|
61
102
|
# Searches the index using the parameters provided in the body hash, following the same
|
62
103
|
# structure Elasticsearch expects.
|
63
104
|
# Returns a DocumentSearch object.
|
64
105
|
def self.search(body)
|
65
|
-
|
106
|
+
search = self.strategy.search(self.document_type, body)
|
107
|
+
Search::DocumentProxy.new(search, self)
|
66
108
|
end
|
67
109
|
|
68
110
|
# Fetches one specific document from the index by ID.
|
69
111
|
def self.get(id)
|
70
|
-
if doc =
|
112
|
+
if doc = self.strategy.get_document(document_type, id)
|
71
113
|
new(doc["_source"].merge(_id: doc['_id']))
|
72
114
|
end
|
73
115
|
end
|
74
116
|
|
75
117
|
# Removes one specific document from the index.
|
76
118
|
def self.delete(id)
|
77
|
-
|
119
|
+
self.strategy.delete_document(document_type, id)
|
78
120
|
end
|
79
121
|
|
80
122
|
# Removes entries based on a search
|
81
123
|
def self.delete_by_search(search)
|
82
|
-
|
124
|
+
self.strategy.delete_by_query(document_type, search.body)
|
83
125
|
end
|
84
126
|
|
85
127
|
# Bulk index the provided documents
|
86
128
|
def self.bulk_index(documents)
|
87
|
-
|
129
|
+
self.strategy.bulk do |b|
|
88
130
|
documents.each do |doc|
|
89
131
|
b.index(self.document_type, doc._id, doc.to_document)
|
90
132
|
end
|
@@ -124,7 +166,15 @@ module Elasticity
|
|
124
166
|
|
125
167
|
# Update this object on the index, creating or updating the document.
|
126
168
|
def update
|
127
|
-
self.class.
|
169
|
+
self._id, @created = self.class.strategy.index_document(self.class.document_type, _id, to_document)
|
170
|
+
end
|
171
|
+
|
172
|
+
def delete
|
173
|
+
self.class.delete(self._id)
|
174
|
+
end
|
175
|
+
|
176
|
+
def created?
|
177
|
+
@created || false
|
128
178
|
end
|
129
179
|
end
|
130
180
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Elasticity
|
2
|
+
class InstrumentedClient
|
3
|
+
def initialize(client)
|
4
|
+
@client = client
|
5
|
+
end
|
6
|
+
|
7
|
+
# Generate wrapper methods for @client.indices
|
8
|
+
%w(exists create delete get_settings get_mapping flush get_alias get_aliases put_alias delete_alias exists_alias update_aliases).each do |method_name|
|
9
|
+
full_name = "index_#{method_name}"
|
10
|
+
|
11
|
+
define_method(full_name) do |*args, &block|
|
12
|
+
instrument(full_name, args) do
|
13
|
+
@client.indices.public_send(method_name, *args, &block)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Generate wrapper methods for @client
|
19
|
+
%w(index delete get mget search msearch scroll delete_by_query bulk).each do |method_name|
|
20
|
+
define_method(method_name) do |*args, &block|
|
21
|
+
instrument(method_name, args) do
|
22
|
+
@client.public_send(method_name, *args, &block)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def instrument(name, args)
|
30
|
+
ActiveSupport::Notifications.instrument("#{name}.elasticity", args: args, backtrace: caller(1)) do
|
31
|
+
yield
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require "active_support/subscriber"
|
2
|
+
require "active_support/log_subscriber"
|
3
|
+
|
4
|
+
module Elasticity
|
5
|
+
GRAY = "\e[90m"
|
6
|
+
|
7
|
+
class LogSubscriber < ::ActiveSupport::LogSubscriber
|
8
|
+
%w(exists create delete get_settings get_mapping flush get_alias get_aliases put_alias delete_alias exists_alias update_aliases).each do |method_name|
|
9
|
+
define_method("index_#{method_name}") do |event|
|
10
|
+
log_event(event)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
%w(index delete get search scroll delete_by_query bulk).each do |method_name|
|
15
|
+
define_method(method_name) do |event|
|
16
|
+
log_event(event)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def multi_search(event)
|
21
|
+
log_event(event)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def log_event(event)
|
27
|
+
bt = event.payload[:backtrace]
|
28
|
+
|
29
|
+
if bt.present? && defined?(Rails)
|
30
|
+
bt = Rails.backtrace_cleaner.clean(bt)
|
31
|
+
end
|
32
|
+
|
33
|
+
message = "#{event.transaction_id} #{event.name} #{"%.2f" % event.duration}ms #{MultiJson.dump(event.payload[:args], pretty: Elasticity.config.pretty_json)}"
|
34
|
+
|
35
|
+
if bt = event.payload[:backtrace]
|
36
|
+
bt = Rails.backtrace_cleaner.clean(bt) if defined?(Rails)
|
37
|
+
lines = bt[0,4].map { |l| color(l, GRAY) }.join("\n")
|
38
|
+
message << "\n#{lines}"
|
39
|
+
end
|
40
|
+
|
41
|
+
debug(message)
|
42
|
+
|
43
|
+
exception, message = event.payload[:exception]
|
44
|
+
if exception
|
45
|
+
error("#{event.transaction_id} #{event.name} ERROR #{exception}: #{message}")
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -7,19 +7,18 @@ module Elasticity
|
|
7
7
|
end
|
8
8
|
|
9
9
|
def add(name, search, documents: nil, active_records: nil)
|
10
|
-
|
11
|
-
when documents && active_records
|
10
|
+
if !documents.nil? && !active_records.nil?
|
12
11
|
raise ArgumentError, "you can only pass either :documents or :active_records as an option"
|
13
|
-
|
14
|
-
Search::DocumentMapper.new(documents)
|
15
|
-
when active_records
|
16
|
-
Search::ActiveRecordMapper.new(active_records)
|
17
|
-
else
|
12
|
+
elsif documents.nil? && active_records.nil?
|
18
13
|
raise ArgumentError, "you need to provide either :documents or :active_records as an option"
|
19
14
|
end
|
20
15
|
|
21
|
-
@searches[name] = {
|
22
|
-
|
16
|
+
@searches[name] = {
|
17
|
+
search_definition: search.search_definition,
|
18
|
+
documents: documents,
|
19
|
+
active_records: active_records
|
20
|
+
}
|
21
|
+
|
23
22
|
name
|
24
23
|
end
|
25
24
|
|
@@ -31,18 +30,26 @@ module Elasticity
|
|
31
30
|
private
|
32
31
|
|
33
32
|
def fetch
|
34
|
-
bodies = @searches.values.map
|
33
|
+
bodies = @searches.values.map do |hsh|
|
34
|
+
hsh[:search_definition].to_msearch_args
|
35
|
+
end
|
35
36
|
|
36
|
-
response = ActiveSupport::Notifications.instrument("multi_search.elasticity", args: { body:
|
37
|
-
Elasticity.config.client.msearch(body: bodies)
|
37
|
+
response = ActiveSupport::Notifications.instrument("multi_search.elasticity", args: { body: bodies }) do
|
38
|
+
Elasticity.config.client.msearch(body: bodies.map(&:dup))
|
38
39
|
end
|
39
40
|
|
40
41
|
results = {}
|
41
42
|
|
42
43
|
@searches.keys.each_with_index do |name, idx|
|
43
|
-
resp
|
44
|
-
|
45
|
-
|
44
|
+
resp = response["responses"][idx]
|
45
|
+
search = @searches[name]
|
46
|
+
|
47
|
+
results[name] = case
|
48
|
+
when search[:documents]
|
49
|
+
resp["hits"]["hits"].map { |hit| search[:documents].from_hit(hit) }
|
50
|
+
when search[:active_records]
|
51
|
+
Search::ActiveRecordProxy.from_hits(search[:active_records], resp["hits"]["hits"])
|
52
|
+
end
|
46
53
|
end
|
47
54
|
|
48
55
|
results
|