elasticsearch-rails-ha 1.0.9 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/elasticsearch/rails/ha/parallel_indexer.rb +20 -10
- data/lib/elasticsearch/rails/ha/version.rb +1 -1
- metadata +15 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da23a4c8436e4f5f8659fd611b0a0980db37bab1
|
4
|
+
data.tar.gz: 9c3b206a17cb6304a515a05f774ed7353b204f95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ec71d34671adb3d7e6416c4a8193c71cda707b55878ac2ea45878c4c0880daadb145e0036abfdc9820156ff34b436f0a4c3046356619183b1b09f71eb6c75075
|
7
|
+
data.tar.gz: 1364788c8bd1436fe6d46ea30fc41d3741c6ff178e2420089c1042468629a8d522e101157c9bde031a4da18a4ea5814e541940a6854e191b2b8319193f7648d9
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# elasticsearch-rails-ha RubyGem
|
2
2
|
|
3
|
-
[![Build Status](https://travis-ci.org/
|
3
|
+
[![Build Status](https://travis-ci.org/karpet/elasticsearch-rails-ha-gem.svg?branch=master)](https://travis-ci.org/karpet/elasticsearch-rails-ha-gem)
|
4
4
|
|
5
5
|
Elasticsearch for Rails, high availability extensions.
|
6
6
|
|
@@ -1,14 +1,24 @@
|
|
1
1
|
require 'active_record/base'
|
2
2
|
require 'ansi'
|
3
|
+
require 'term/ansicolor'
|
3
4
|
require 'pp'
|
4
5
|
|
5
6
|
module Elasticsearch
|
6
7
|
module Rails
|
7
8
|
module HA
|
8
9
|
class ParallelIndexer
|
10
|
+
include Term::ANSIColor
|
9
11
|
|
10
12
|
attr_reader :klass, :idx_name, :nprocs, :batch_size, :max, :force, :verbose, :scope
|
11
13
|
|
14
|
+
def blue_log(msg)
|
15
|
+
blue{ msg }
|
16
|
+
end
|
17
|
+
|
18
|
+
def red_log(msg)
|
19
|
+
red{ msg }
|
20
|
+
end
|
21
|
+
|
12
22
|
# leverage multiple cores to run indexing in parallel
|
13
23
|
def initialize(opts)
|
14
24
|
@klass = opts[:klass] or fail "klass required"
|
@@ -41,11 +51,11 @@ module Elasticsearch
|
|
41
51
|
offsets.push( chunk.first )
|
42
52
|
end
|
43
53
|
if @verbose
|
44
|
-
puts
|
54
|
+
puts blue_log("Parallel Indexer: index=#{@idx_name} total=#{@total_expected} nprocs=#{@nprocs} pool_size=#{@pool_size} offsets=#{offsets} ")
|
45
55
|
end
|
46
56
|
|
47
57
|
if @force
|
48
|
-
@verbose and puts
|
58
|
+
@verbose and puts blue_log("Force creating new index")
|
49
59
|
klass.__elasticsearch__.create_index! force: true, index: idx_name
|
50
60
|
klass.__elasticsearch__.refresh_index! index: idx_name
|
51
61
|
end
|
@@ -85,25 +95,25 @@ module Elasticsearch
|
|
85
95
|
pstat = pair[1]
|
86
96
|
exit_ok = true
|
87
97
|
if pstat.exited?
|
88
|
-
@verbose and puts
|
98
|
+
@verbose and puts blue_log("PID #{pid} exited with #{pstat.exitstatus}")
|
89
99
|
end
|
90
100
|
if pstat.signaled?
|
91
|
-
puts
|
101
|
+
puts red_log(" >> #{pid} exited with uncaught signal #{pstat.termsig}")
|
92
102
|
exit_ok = false
|
93
103
|
end
|
94
104
|
|
95
105
|
if !pstat.success?
|
96
|
-
puts
|
106
|
+
puts red_log(" >> #{pid} was not successful")
|
97
107
|
exit_ok = false
|
98
108
|
end
|
99
109
|
|
100
110
|
if pair[1].exitstatus != 0
|
101
|
-
puts
|
111
|
+
puts red_log(" >> #{pid} exited with non-zero status")
|
102
112
|
exit_ok = false
|
103
113
|
end
|
104
114
|
|
105
115
|
if !exit_ok
|
106
|
-
raise
|
116
|
+
raise red_log("PID #{pair[0]} exited abnormally, so the whole reindex fails")
|
107
117
|
end
|
108
118
|
end
|
109
119
|
end
|
@@ -117,7 +127,7 @@ module Elasticsearch
|
|
117
127
|
|
118
128
|
completed = 0
|
119
129
|
errors = []
|
120
|
-
@verbose and puts
|
130
|
+
@verbose and puts blue_log("Start worker #{$$} at offset #{start_at}")
|
121
131
|
pbar = ::ANSI::Progressbar.new("#{klass} [#{$$}]", @pool_size, STDOUT) rescue nil
|
122
132
|
checkpoint = false
|
123
133
|
if pbar
|
@@ -142,7 +152,7 @@ module Elasticsearch
|
|
142
152
|
pbar.inc resp['items'].size
|
143
153
|
end
|
144
154
|
if checkpoint && @verbose
|
145
|
-
puts
|
155
|
+
puts blue_log("[#{$$}] #{Time.now.utc.iso8601} : #{completed} records completed")
|
146
156
|
end
|
147
157
|
STDERR.flush
|
148
158
|
STDOUT.flush
|
@@ -152,7 +162,7 @@ module Elasticsearch
|
|
152
162
|
end
|
153
163
|
if completed >= @pool_size || (@max && @max.to_i == completed)
|
154
164
|
pbar.finish if pbar
|
155
|
-
@verbose and puts
|
165
|
+
@verbose and puts blue_log("Worker #{$$} finished #{completed} records")
|
156
166
|
exit!(true) # exit child worker
|
157
167
|
end
|
158
168
|
end # end do |resp| block
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticsearch-rails-ha
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Karman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: elasticsearch-model
|
@@ -42,16 +42,16 @@ dependencies:
|
|
42
42
|
name: elasticsearch-indexstager
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 1.1.3
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 1.1.3
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: ansi
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,13 +67,13 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: term-ansicolor
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
|
-
type: :
|
76
|
+
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
@@ -112,16 +112,16 @@ dependencies:
|
|
112
112
|
name: rspec
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- - "
|
115
|
+
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
117
|
+
version: '3.0'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - "
|
122
|
+
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
124
|
+
version: '3.0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: elasticsearch-extensions
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -166,7 +166,7 @@ dependencies:
|
|
166
166
|
version: '3.1'
|
167
167
|
description: High Availability extensions to the Elasticsearch::Rails gem
|
168
168
|
email:
|
169
|
-
-
|
169
|
+
- karpet@peknet.com
|
170
170
|
executables: []
|
171
171
|
extensions: []
|
172
172
|
extra_rdoc_files: []
|
@@ -179,7 +179,7 @@ files:
|
|
179
179
|
- lib/elasticsearch/rails/ha/parallel_indexer.rb
|
180
180
|
- lib/elasticsearch/rails/ha/tasks.rb
|
181
181
|
- lib/elasticsearch/rails/ha/version.rb
|
182
|
-
homepage: https://github.com/
|
182
|
+
homepage: https://github.com/karpet/elasticsearch-rails-ha-gem
|
183
183
|
licenses:
|
184
184
|
- CC0
|
185
185
|
metadata: {}
|
@@ -199,9 +199,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
199
|
version: '0'
|
200
200
|
requirements: []
|
201
201
|
rubyforge_project:
|
202
|
-
rubygems_version: 2.
|
202
|
+
rubygems_version: 2.6.13
|
203
203
|
signing_key:
|
204
204
|
specification_version: 4
|
205
205
|
summary: High Availability extensions to the Elasticsearch::Rails gem
|
206
206
|
test_files: []
|
207
|
-
has_rdoc:
|