elasticated 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/elasticated.rb +0 -2
- data/lib/elasticated/client.rb +23 -14
- data/lib/elasticated/configuration.rb +1 -0
- data/lib/elasticated/repository.rb +5 -4
- data/lib/version.rb +5 -1
- metadata +2 -5
- data/lib/elasticated/repositories/monthly_partitioned_repository.rb +0 -96
- data/spec/monthly_partitioned_repository_spec.rb +0 -99
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7828418b23833264f77949ef5c083c9ac8d5970d
|
4
|
+
data.tar.gz: ba7887d592b1f2643821fef99f172303168967e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9262818290a6679e8997f3784d0de0fcbac4b282f64ee0e43b7800dba72611b5b30a2d3eb13a8c15a2a8f20aa921550dab512e4441e47a5332defb93c13b2175
|
7
|
+
data.tar.gz: 0a520739e691bc59e8613816a3ae038785828ff4cbc8bb0c0dc7c33ee165674711e9183496152aa93aa55e38379009abcf21f87fbebd0b8839266a8f99534af0
|
data/lib/elasticated.rb
CHANGED
@@ -90,8 +90,6 @@ require_relative 'elasticated/mapping/builder'
|
|
90
90
|
require_relative 'elasticated/index_selector'
|
91
91
|
require_relative 'elasticated/partitioned_repository'
|
92
92
|
|
93
|
-
require_relative 'elasticated/repositories/monthly_partitioned_repository'
|
94
|
-
|
95
93
|
|
96
94
|
module Elasticated
|
97
95
|
|
data/lib/elasticated/client.rb
CHANGED
@@ -9,11 +9,13 @@ module Elasticated
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def index_document(document_source, opts={})
|
12
|
-
transport.index opts.merge body: document_source
|
12
|
+
response = transport.index opts.merge body: document_source
|
13
|
+
log.debug "Document indexed #{response.to_json}"
|
13
14
|
end
|
14
15
|
|
15
16
|
def update_document(document_source, opts={})
|
16
|
-
transport.update opts.merge body: { doc: document_source }
|
17
|
+
response = transport.update opts.merge body: { doc: document_source }
|
18
|
+
log.debug "Document updated #{response.to_json}"
|
17
19
|
end
|
18
20
|
|
19
21
|
def index_exists?(index_name)
|
@@ -28,32 +30,31 @@ module Elasticated
|
|
28
30
|
body.merge!(settings: { number_of_shards: shards }) if shards
|
29
31
|
body.merge!(mappings: mapping) if mapping
|
30
32
|
args.merge! body: body unless body.empty?
|
31
|
-
log.info "Creating index #{index_name}"
|
33
|
+
log.info "Creating index '#{index_name}'"
|
32
34
|
transport.indices.create args
|
33
|
-
log.info "Index #{index_name} created"
|
35
|
+
log.info "Index '#{index_name}' created"
|
34
36
|
end
|
35
37
|
|
36
38
|
def create_alias(index_name, new_alias)
|
37
|
-
log.info "Putting alias #{new_alias} to index #{index_name}"
|
38
39
|
transport.indices.put_alias index: index_name, name: new_alias
|
39
|
-
log.info "Alias #{new_alias} for index #{index_name} created"
|
40
|
+
log.info "Alias '#{new_alias}' for index '#{index_name}' created"
|
40
41
|
end
|
41
42
|
|
42
43
|
def remove_aliases(index_name)
|
43
44
|
aliases = transport.indices.get_aliases[index_name]['aliases'].map(&:first) rescue Array.new
|
44
|
-
aliases.each do |
|
45
|
-
|
45
|
+
aliases.each do |index_alias|
|
46
|
+
remove_alias index_name, index_alias
|
46
47
|
end
|
47
48
|
end
|
48
49
|
|
49
50
|
def remove_alias(index_name, index_alias)
|
50
|
-
log.info "Alias #{index_alias} removed from index #{index_name}"
|
51
51
|
transport.indices.delete_alias index: index_name, name: index_alias
|
52
|
+
log.info "Alias '#{index_alias}' removed from index '#{index_name}'"
|
52
53
|
end
|
53
54
|
|
54
55
|
def refresh_index(index_name)
|
55
|
-
log.debug "Refreshing #{index_name}"
|
56
56
|
transport.indices.refresh index: index_name
|
57
|
+
log.debug "Index '#{index_name}' refreshed"
|
57
58
|
end
|
58
59
|
|
59
60
|
def refresh
|
@@ -61,24 +62,32 @@ module Elasticated
|
|
61
62
|
end
|
62
63
|
|
63
64
|
def count(body, opts={})
|
64
|
-
|
65
|
+
log_query :count, body.to_json, opts
|
65
66
|
transport.count opts.merge body: body
|
66
67
|
end
|
67
68
|
|
68
69
|
def search(body, opts={})
|
69
|
-
|
70
|
+
log_query :search, body.to_json, opts
|
70
71
|
transport.search opts.merge body: body
|
71
72
|
end
|
72
73
|
|
73
74
|
def scroll(scroll_id, opts={})
|
74
|
-
|
75
|
+
log_query :scroll, scroll_id, opts
|
75
76
|
transport.scroll opts.merge scroll_id: scroll_id
|
76
77
|
end
|
77
78
|
|
78
79
|
def delete(body, opts={})
|
79
|
-
|
80
|
+
log_query :delete, body.to_json, opts
|
80
81
|
transport.delete_by_query opts.merge body: body
|
81
82
|
end
|
82
83
|
|
84
|
+
protected
|
85
|
+
|
86
|
+
def log_query(query_type, message_body, query_opts={})
|
87
|
+
message = "Elasticsearch #{query_type} #{message_body}"
|
88
|
+
message = "#{message} with opts #{query_opts.to_json}" unless query_opts.empty?
|
89
|
+
log.debug message
|
90
|
+
end
|
91
|
+
|
83
92
|
end
|
84
93
|
end
|
@@ -90,14 +90,13 @@ module Elasticated
|
|
90
90
|
end
|
91
91
|
|
92
92
|
def _exec_paginated_search(query, aggregated, opts={})
|
93
|
-
scroll_interval = '3m'
|
94
93
|
body = aggregated ? query.build_for_aggregated_search : query.build_for_search
|
95
94
|
if query.limited?
|
96
95
|
response = client.search body, opts
|
97
96
|
Results.from_elasticsearch_response response, query
|
98
97
|
elsif query.sorted? || query.aggregated?
|
99
98
|
# normal pagination
|
100
|
-
page_size = 50
|
99
|
+
page_size = Configuration.scroll_page_size || 50
|
101
100
|
current_page = 1
|
102
101
|
loop do
|
103
102
|
offset = page_size * (current_page - 1)
|
@@ -115,8 +114,10 @@ module Elasticated
|
|
115
114
|
results
|
116
115
|
else
|
117
116
|
# scan & scroll
|
118
|
-
|
119
|
-
|
117
|
+
scroll_expiration_time = Configuration.scroll_expiration_time || '3m'
|
118
|
+
page_size = Configuration.scroll_page_size || 1000
|
119
|
+
response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_expiration_time, size: page_size)
|
120
|
+
while response = client.scroll(response['_scroll_id'], scroll: scroll_expiration_time) and not response['hits']['hits'].empty? do
|
120
121
|
results = results ? results.append_results_from(response) : Results.from_elasticsearch_response(response, query)
|
121
122
|
end
|
122
123
|
results ? results : Results.from_elasticsearch_response(response, query)
|
data/lib/version.rb
CHANGED
@@ -1,9 +1,13 @@
|
|
1
1
|
module Elasticated
|
2
|
-
VERSION = '1.
|
2
|
+
VERSION = '1.1.0'
|
3
3
|
end
|
4
4
|
|
5
5
|
# Changelog
|
6
6
|
|
7
|
+
# 1.1.0
|
8
|
+
# Se elimina MonthlyPartitionedRepository cediendo paso a PartitionedRepository
|
9
|
+
# Ahora el scroll expiration time y los tamanos de pagina son configurables
|
10
|
+
|
7
11
|
# 1.0.0
|
8
12
|
# Cambios en Repository
|
9
13
|
# Se elimina InsightsRepository y sus hijos
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticated
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Fernandez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -162,7 +162,6 @@ files:
|
|
162
162
|
- lib/elasticated/query.rb
|
163
163
|
- lib/elasticated/query_aggregations.rb
|
164
164
|
- lib/elasticated/query_conditions.rb
|
165
|
-
- lib/elasticated/repositories/monthly_partitioned_repository.rb
|
166
165
|
- lib/elasticated/repository.rb
|
167
166
|
- lib/elasticated/results.rb
|
168
167
|
- lib/version.rb
|
@@ -175,7 +174,6 @@ files:
|
|
175
174
|
- spec/elasticsearch_top_hits_response.json
|
176
175
|
- spec/integration_spec.rb
|
177
176
|
- spec/mapping_spec.rb
|
178
|
-
- spec/monthly_partitioned_repository_spec.rb
|
179
177
|
- spec/query_aggregations_spec.rb
|
180
178
|
- spec/query_conditions_spec.rb
|
181
179
|
- spec/query_spec.rb
|
@@ -216,7 +214,6 @@ test_files:
|
|
216
214
|
- spec/elasticsearch_top_hits_response.json
|
217
215
|
- spec/integration_spec.rb
|
218
216
|
- spec/mapping_spec.rb
|
219
|
-
- spec/monthly_partitioned_repository_spec.rb
|
220
217
|
- spec/query_aggregations_spec.rb
|
221
218
|
- spec/query_conditions_spec.rb
|
222
219
|
- spec/query_spec.rb
|
@@ -1,96 +0,0 @@
|
|
1
|
-
module Elasticated
|
2
|
-
class MonthlyPartitionedRepository < Repository
|
3
|
-
include Configurable
|
4
|
-
|
5
|
-
attr_accessor :_index_name, :_index_alias, :_general_alias
|
6
|
-
attr_accessor :date_field
|
7
|
-
attr_accessor :dynamic_creation, :index_options
|
8
|
-
|
9
|
-
def initialize(opts={})
|
10
|
-
super opts
|
11
|
-
self._index_name = opts[:index_name] || raise('An index prefix should be specified')
|
12
|
-
self._index_alias = opts[:index_alias] || raise('An index alias prefix should be specified')
|
13
|
-
self._general_alias = opts[:general_alias] || _index_alias
|
14
|
-
self.date_field = opts[:date_field] || :date
|
15
|
-
self.index_options = opts[:index_options] # hash with mapping, shards info, etc or nil
|
16
|
-
self.dynamic_creation = opts[:dynamic_creation] || false
|
17
|
-
end
|
18
|
-
|
19
|
-
def execute(action, query, opts={})
|
20
|
-
index_alias = index_alias_for query
|
21
|
-
super action, query, opts.merge(index: index_alias)
|
22
|
-
# rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
|
23
|
-
# action == :count ? 0 : nil
|
24
|
-
end
|
25
|
-
|
26
|
-
def prepare(action, document, opts={})
|
27
|
-
date = date_from document
|
28
|
-
raise("The document has not a valid '#{date_field}' field") unless date
|
29
|
-
check_or_create_index! date
|
30
|
-
super action, document, opts.merge(index: index_alias_for(date))
|
31
|
-
end
|
32
|
-
|
33
|
-
def date_from(document)
|
34
|
-
str = document.source[date_field.to_s]
|
35
|
-
str && Date.parse(str)
|
36
|
-
end
|
37
|
-
|
38
|
-
def check_or_create_index! date
|
39
|
-
index_alias = index_alias_for date
|
40
|
-
return if client.index_exists? index_alias
|
41
|
-
raise("Index '#{index_alias}' not found (dynamic per-month index creation is disabled)") unless dynamic_creation
|
42
|
-
index_name = index_name_for date
|
43
|
-
create_index index_name
|
44
|
-
create_alias index_name, index_alias
|
45
|
-
end
|
46
|
-
|
47
|
-
def create_index! date
|
48
|
-
create_index index_name_for date
|
49
|
-
end
|
50
|
-
|
51
|
-
def create_alias! date
|
52
|
-
index_name = index_name_for date
|
53
|
-
index_alias = index_alias_for date
|
54
|
-
create_alias index_name, index_alias
|
55
|
-
end
|
56
|
-
|
57
|
-
def index_name_for(object)
|
58
|
-
object.is_a?(Query) ? index_alias_for_query(object) : index_name_for_date(object)
|
59
|
-
end
|
60
|
-
|
61
|
-
def index_alias_for(object)
|
62
|
-
object.is_a?(Query) ? index_alias_for_query(object) : index_alias_for_date(object)
|
63
|
-
end
|
64
|
-
|
65
|
-
protected
|
66
|
-
|
67
|
-
def create_index(index_name)
|
68
|
-
client.create_index index_name, index_options
|
69
|
-
log.info "Index #{index_name} created"
|
70
|
-
end
|
71
|
-
|
72
|
-
def create_alias(index_name, index_alias)
|
73
|
-
client.create_alias index_name, index_alias
|
74
|
-
client.create_alias index_name, _general_alias
|
75
|
-
log.info "Alias #{index_alias} created over index #{index_name}"
|
76
|
-
end
|
77
|
-
|
78
|
-
def index_alias_for_query(query)
|
79
|
-
_general_alias # TODO
|
80
|
-
end
|
81
|
-
|
82
|
-
def index_alias_for_date(date)
|
83
|
-
generate_name _index_alias, date
|
84
|
-
end
|
85
|
-
|
86
|
-
def index_name_for_date(date)
|
87
|
-
generate_name _index_name, date
|
88
|
-
end
|
89
|
-
|
90
|
-
def generate_name(prefix, date)
|
91
|
-
str = date.strftime '%Y-%m'
|
92
|
-
"#{prefix}-#{str}" # prefix-YYYY-MM
|
93
|
-
end
|
94
|
-
|
95
|
-
end
|
96
|
-
end
|
@@ -1,99 +0,0 @@
|
|
1
|
-
require_relative 'spec_helper'
|
2
|
-
|
3
|
-
module Elasticated
|
4
|
-
describe MonthlyPartitionedRepository do
|
5
|
-
|
6
|
-
let :repository do
|
7
|
-
MonthlyPartitionedRepository.new index_name: 'fbinsights-v3', index_alias: 'fbinsights'
|
8
|
-
end
|
9
|
-
|
10
|
-
before :each do
|
11
|
-
Elasticated.configure do |config|
|
12
|
-
config.logger = double.as_null_object
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
it "should return the correct index name" do
|
17
|
-
date = Date.parse '2015-09-04'
|
18
|
-
name = repository.index_name_for date
|
19
|
-
expect(name).to eq 'fbinsights-v3-2015-09'
|
20
|
-
end
|
21
|
-
|
22
|
-
it "should return the correct index alias" do
|
23
|
-
date = Date.parse '2015-09-04'
|
24
|
-
name = repository.index_alias_for date
|
25
|
-
expect(name).to eq 'fbinsights-2015-09'
|
26
|
-
end
|
27
|
-
|
28
|
-
context "on a search" do
|
29
|
-
|
30
|
-
let :query do
|
31
|
-
Query.new # TODO date
|
32
|
-
end
|
33
|
-
|
34
|
-
it "should execute a search on the correct index" do
|
35
|
-
expect(repository).to receive(:_exec_search).with query, index: 'fbinsights' # TODO
|
36
|
-
repository.execute_search query
|
37
|
-
end
|
38
|
-
|
39
|
-
it "should execute a count on the correct index" do
|
40
|
-
expect(repository).to receive(:_exec_count).with query, index: 'fbinsights' # TODO
|
41
|
-
repository.execute_count query
|
42
|
-
end
|
43
|
-
|
44
|
-
it "should execute a delete on the correct index" do
|
45
|
-
expect(repository).to receive(:_exec_delete).with query, index: 'fbinsights' # TODO
|
46
|
-
repository.delete_by query
|
47
|
-
end
|
48
|
-
|
49
|
-
end
|
50
|
-
|
51
|
-
context "on an indexation" do
|
52
|
-
|
53
|
-
let :document do
|
54
|
-
Document.create source: { 'date' => '2015-09-04' }
|
55
|
-
end
|
56
|
-
|
57
|
-
it "should use the appropiated index alias" do
|
58
|
-
client = repository.client
|
59
|
-
expect(client).to receive(:index_exists?).with('fbinsights-2015-09').and_return true
|
60
|
-
expect(client).to receive(:index_document).with document.source, index: 'fbinsights-2015-09', type: 'post'
|
61
|
-
repository.index_document document, type: 'post'
|
62
|
-
end
|
63
|
-
|
64
|
-
it "should create the appropiated index (only if dynamic creation is enabled)" do
|
65
|
-
client = repository.client
|
66
|
-
expect(client).to receive(:index_exists?).with('fbinsights-2015-09').and_return false
|
67
|
-
expect{ repository.index_document document, type: 'post' }.to raise_error
|
68
|
-
expect(client).to receive(:index_exists?).with('fbinsights-2015-09').and_return false
|
69
|
-
expect(client).to receive(:create_index).with 'fbinsights-v3-2015-09', anything
|
70
|
-
repository.dynamic_creation = true
|
71
|
-
expect(client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights-2015-09').once
|
72
|
-
expect(client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights').once
|
73
|
-
expect(client).to receive(:index_document).with document.source, index: 'fbinsights-2015-09', type: 'post'
|
74
|
-
repository.index_document document, type: 'post'
|
75
|
-
end
|
76
|
-
|
77
|
-
end
|
78
|
-
|
79
|
-
context "the index creation method" do
|
80
|
-
|
81
|
-
let :date do
|
82
|
-
Date.parse '2015-09-04'
|
83
|
-
end
|
84
|
-
|
85
|
-
it "creates the appropiated index" do
|
86
|
-
expect(repository.client).to receive(:create_index).with 'fbinsights-v3-2015-09', anything
|
87
|
-
repository.create_index! date
|
88
|
-
end
|
89
|
-
|
90
|
-
it "creates the appropiated alias" do
|
91
|
-
expect(repository.client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights-2015-09').once
|
92
|
-
expect(repository.client).to receive(:create_alias).with('fbinsights-v3-2015-09', 'fbinsights').once
|
93
|
-
repository.create_alias! date
|
94
|
-
end
|
95
|
-
|
96
|
-
end
|
97
|
-
|
98
|
-
end
|
99
|
-
end
|