elasticsearch-rails-ha 1.0.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/elasticsearch/rails/ha/parallel_indexer.rb +20 -10
- data/lib/elasticsearch/rails/ha/version.rb +1 -1
- metadata +15 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da23a4c8436e4f5f8659fd611b0a0980db37bab1
|
4
|
+
data.tar.gz: 9c3b206a17cb6304a515a05f774ed7353b204f95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ec71d34671adb3d7e6416c4a8193c71cda707b55878ac2ea45878c4c0880daadb145e0036abfdc9820156ff34b436f0a4c3046356619183b1b09f71eb6c75075
|
7
|
+
data.tar.gz: 1364788c8bd1436fe6d46ea30fc41d3741c6ff178e2420089c1042468629a8d522e101157c9bde031a4da18a4ea5814e541940a6854e191b2b8319193f7648d9
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# elasticsearch-rails-ha RubyGem
|
2
2
|
|
3
|
-
[](https://travis-ci.org/karpet/elasticsearch-rails-ha-gem)
|
4
4
|
|
5
5
|
Elasticsearch for Rails, high availability extensions.
|
6
6
|
|
@@ -1,14 +1,24 @@
|
|
1
1
|
require 'active_record/base'
|
2
2
|
require 'ansi'
|
3
|
+
require 'term/ansicolor'
|
3
4
|
require 'pp'
|
4
5
|
|
5
6
|
module Elasticsearch
|
6
7
|
module Rails
|
7
8
|
module HA
|
8
9
|
class ParallelIndexer
|
10
|
+
include Term::ANSIColor
|
9
11
|
|
10
12
|
attr_reader :klass, :idx_name, :nprocs, :batch_size, :max, :force, :verbose, :scope
|
11
13
|
|
14
|
+
def blue_log(msg)
|
15
|
+
blue{ msg }
|
16
|
+
end
|
17
|
+
|
18
|
+
def red_log(msg)
|
19
|
+
red{ msg }
|
20
|
+
end
|
21
|
+
|
12
22
|
# leverage multiple cores to run indexing in parallel
|
13
23
|
def initialize(opts)
|
14
24
|
@klass = opts[:klass] or fail "klass required"
|
@@ -41,11 +51,11 @@ module Elasticsearch
|
|
41
51
|
offsets.push( chunk.first )
|
42
52
|
end
|
43
53
|
if @verbose
|
44
|
-
puts
|
54
|
+
puts blue_log("Parallel Indexer: index=#{@idx_name} total=#{@total_expected} nprocs=#{@nprocs} pool_size=#{@pool_size} offsets=#{offsets} ")
|
45
55
|
end
|
46
56
|
|
47
57
|
if @force
|
48
|
-
@verbose and puts
|
58
|
+
@verbose and puts blue_log("Force creating new index")
|
49
59
|
klass.__elasticsearch__.create_index! force: true, index: idx_name
|
50
60
|
klass.__elasticsearch__.refresh_index! index: idx_name
|
51
61
|
end
|
@@ -85,25 +95,25 @@ module Elasticsearch
|
|
85
95
|
pstat = pair[1]
|
86
96
|
exit_ok = true
|
87
97
|
if pstat.exited?
|
88
|
-
@verbose and puts
|
98
|
+
@verbose and puts blue_log("PID #{pid} exited with #{pstat.exitstatus}")
|
89
99
|
end
|
90
100
|
if pstat.signaled?
|
91
|
-
puts
|
101
|
+
puts red_log(" >> #{pid} exited with uncaught signal #{pstat.termsig}")
|
92
102
|
exit_ok = false
|
93
103
|
end
|
94
104
|
|
95
105
|
if !pstat.success?
|
96
|
-
puts
|
106
|
+
puts red_log(" >> #{pid} was not successful")
|
97
107
|
exit_ok = false
|
98
108
|
end
|
99
109
|
|
100
110
|
if pair[1].exitstatus != 0
|
101
|
-
puts
|
111
|
+
puts red_log(" >> #{pid} exited with non-zero status")
|
102
112
|
exit_ok = false
|
103
113
|
end
|
104
114
|
|
105
115
|
if !exit_ok
|
106
|
-
raise
|
116
|
+
raise red_log("PID #{pair[0]} exited abnormally, so the whole reindex fails")
|
107
117
|
end
|
108
118
|
end
|
109
119
|
end
|
@@ -117,7 +127,7 @@ module Elasticsearch
|
|
117
127
|
|
118
128
|
completed = 0
|
119
129
|
errors = []
|
120
|
-
@verbose and puts
|
130
|
+
@verbose and puts blue_log("Start worker #{$$} at offset #{start_at}")
|
121
131
|
pbar = ::ANSI::Progressbar.new("#{klass} [#{$$}]", @pool_size, STDOUT) rescue nil
|
122
132
|
checkpoint = false
|
123
133
|
if pbar
|
@@ -142,7 +152,7 @@ module Elasticsearch
|
|
142
152
|
pbar.inc resp['items'].size
|
143
153
|
end
|
144
154
|
if checkpoint && @verbose
|
145
|
-
puts
|
155
|
+
puts blue_log("[#{$$}] #{Time.now.utc.iso8601} : #{completed} records completed")
|
146
156
|
end
|
147
157
|
STDERR.flush
|
148
158
|
STDOUT.flush
|
@@ -152,7 +162,7 @@ module Elasticsearch
|
|
152
162
|
end
|
153
163
|
if completed >= @pool_size || (@max && @max.to_i == completed)
|
154
164
|
pbar.finish if pbar
|
155
|
-
@verbose and puts
|
165
|
+
@verbose and puts blue_log("Worker #{$$} finished #{completed} records")
|
156
166
|
exit!(true) # exit child worker
|
157
167
|
end
|
158
168
|
end # end do |resp| block
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticsearch-rails-ha
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Karman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: elasticsearch-model
|
@@ -42,16 +42,16 @@ dependencies:
|
|
42
42
|
name: elasticsearch-indexstager
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 1.1.3
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 1.1.3
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: ansi
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,13 +67,13 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: term-ansicolor
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
|
-
type: :
|
76
|
+
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
@@ -112,16 +112,16 @@ dependencies:
|
|
112
112
|
name: rspec
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- - "
|
115
|
+
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
117
|
+
version: '3.0'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - "
|
122
|
+
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
124
|
+
version: '3.0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: elasticsearch-extensions
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -166,7 +166,7 @@ dependencies:
|
|
166
166
|
version: '3.1'
|
167
167
|
description: High Availability extensions to the Elasticsearch::Rails gem
|
168
168
|
email:
|
169
|
-
-
|
169
|
+
- karpet@peknet.com
|
170
170
|
executables: []
|
171
171
|
extensions: []
|
172
172
|
extra_rdoc_files: []
|
@@ -179,7 +179,7 @@ files:
|
|
179
179
|
- lib/elasticsearch/rails/ha/parallel_indexer.rb
|
180
180
|
- lib/elasticsearch/rails/ha/tasks.rb
|
181
181
|
- lib/elasticsearch/rails/ha/version.rb
|
182
|
-
homepage: https://github.com/
|
182
|
+
homepage: https://github.com/karpet/elasticsearch-rails-ha-gem
|
183
183
|
licenses:
|
184
184
|
- CC0
|
185
185
|
metadata: {}
|
@@ -199,9 +199,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
199
|
version: '0'
|
200
200
|
requirements: []
|
201
201
|
rubyforge_project:
|
202
|
-
rubygems_version: 2.
|
202
|
+
rubygems_version: 2.6.13
|
203
203
|
signing_key:
|
204
204
|
specification_version: 4
|
205
205
|
summary: High Availability extensions to the Elasticsearch::Rails gem
|
206
206
|
test_files: []
|
207
|
-
has_rdoc:
|