promiscuous 0.92.0 → 0.100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/promiscuous.rb +2 -5
- data/lib/promiscuous/amqp.rb +1 -2
- data/lib/promiscuous/cli.rb +3 -43
- data/lib/promiscuous/config.rb +5 -7
- data/lib/promiscuous/error/dependency.rb +1 -3
- data/lib/promiscuous/publisher/context.rb +1 -1
- data/lib/promiscuous/publisher/context/base.rb +3 -34
- data/lib/promiscuous/publisher/model/base.rb +5 -25
- data/lib/promiscuous/publisher/model/mock.rb +5 -7
- data/lib/promiscuous/publisher/operation/active_record.rb +4 -69
- data/lib/promiscuous/publisher/operation/atomic.rb +1 -3
- data/lib/promiscuous/publisher/operation/base.rb +33 -123
- data/lib/promiscuous/publisher/operation/mongoid.rb +0 -67
- data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -1
- data/lib/promiscuous/publisher/operation/transaction.rb +1 -3
- data/lib/promiscuous/railtie.rb +0 -31
- data/lib/promiscuous/subscriber.rb +1 -1
- data/lib/promiscuous/subscriber/{worker/message.rb → message.rb} +12 -40
- data/lib/promiscuous/subscriber/model/active_record.rb +1 -1
- data/lib/promiscuous/subscriber/model/base.rb +4 -4
- data/lib/promiscuous/subscriber/model/mongoid.rb +3 -3
- data/lib/promiscuous/subscriber/operation.rb +74 -3
- data/lib/promiscuous/subscriber/unit_of_work.rb +110 -0
- data/lib/promiscuous/subscriber/worker.rb +3 -7
- data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +2 -6
- data/lib/promiscuous/subscriber/worker/pump.rb +2 -11
- data/lib/promiscuous/version.rb +1 -1
- metadata +18 -36
- data/lib/promiscuous/error/missing_context.rb +0 -29
- data/lib/promiscuous/publisher/bootstrap.rb +0 -27
- data/lib/promiscuous/publisher/bootstrap/connection.rb +0 -25
- data/lib/promiscuous/publisher/bootstrap/data.rb +0 -127
- data/lib/promiscuous/publisher/bootstrap/mode.rb +0 -19
- data/lib/promiscuous/publisher/bootstrap/status.rb +0 -40
- data/lib/promiscuous/publisher/bootstrap/version.rb +0 -46
- data/lib/promiscuous/publisher/context/middleware.rb +0 -94
- data/lib/promiscuous/resque.rb +0 -12
- data/lib/promiscuous/sidekiq.rb +0 -15
- data/lib/promiscuous/subscriber/message_processor.rb +0 -4
- data/lib/promiscuous/subscriber/message_processor/base.rb +0 -54
- data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +0 -17
- data/lib/promiscuous/subscriber/message_processor/regular.rb +0 -238
- data/lib/promiscuous/subscriber/operation/base.rb +0 -66
- data/lib/promiscuous/subscriber/operation/bootstrap.rb +0 -60
- data/lib/promiscuous/subscriber/operation/regular.rb +0 -19
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +0 -333
@@ -14,21 +14,12 @@ class Promiscuous::Subscriber::Worker::Pump
|
|
14
14
|
options[:bindings][exchange] = ['*']
|
15
15
|
end
|
16
16
|
|
17
|
-
if Promiscuous::Config.bootstrap
|
18
|
-
options[:bindings][Promiscuous::AMQP::BOOTSTRAP_EXCHANGE] = ['*']
|
19
|
-
end
|
20
|
-
|
21
17
|
subscribe(options, &method(:on_message))
|
22
18
|
end
|
23
19
|
|
24
20
|
def on_message(metadata, payload)
|
25
|
-
msg = Promiscuous::Subscriber::
|
26
|
-
|
27
|
-
# Bootstrapping doesn't require synchronzation
|
28
|
-
@root.runner.messages_to_process << msg
|
29
|
-
else
|
30
|
-
@root.message_synchronizer.process_when_ready(msg)
|
31
|
-
end
|
21
|
+
msg = Promiscuous::Subscriber::Message.new(payload, :metadata => metadata, :root_worker => @root)
|
22
|
+
@root.runner.messages_to_process << msg
|
32
23
|
rescue Exception => e
|
33
24
|
Promiscuous.warn "[receive] cannot process message: #{e}\n#{e.backtrace.join("\n")}"
|
34
25
|
Promiscuous::Config.error_notifier.call(e)
|
data/lib/promiscuous/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: promiscuous
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.100.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicolas Viennot
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-02
|
12
|
+
date: 2014-04-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -137,7 +137,7 @@ dependencies:
|
|
137
137
|
- - ~>
|
138
138
|
- !ruby/object:Gem::Version
|
139
139
|
version: 1.8.0
|
140
|
-
description: Replicate
|
140
|
+
description: Replicate models across applications
|
141
141
|
email:
|
142
142
|
- nicolas@viennot.biz
|
143
143
|
- kareem@doubleonemedia.com
|
@@ -146,52 +146,40 @@ executables:
|
|
146
146
|
extensions: []
|
147
147
|
extra_rdoc_files: []
|
148
148
|
files:
|
149
|
-
- bin/promiscuous
|
150
|
-
- lib/promiscuous.rb
|
151
|
-
- lib/promiscuous/amqp.rb
|
152
149
|
- lib/promiscuous/amqp/bunny.rb
|
153
150
|
- lib/promiscuous/amqp/fake.rb
|
154
151
|
- lib/promiscuous/amqp/file.rb
|
155
152
|
- lib/promiscuous/amqp/hot_bunnies.rb
|
156
153
|
- lib/promiscuous/amqp/null.rb
|
154
|
+
- lib/promiscuous/amqp.rb
|
157
155
|
- lib/promiscuous/autoload.rb
|
158
156
|
- lib/promiscuous/cli.rb
|
159
157
|
- lib/promiscuous/config.rb
|
160
158
|
- lib/promiscuous/convenience.rb
|
161
159
|
- lib/promiscuous/dependency.rb
|
162
160
|
- lib/promiscuous/dsl.rb
|
163
|
-
- lib/promiscuous/error.rb
|
164
161
|
- lib/promiscuous/error/base.rb
|
165
162
|
- lib/promiscuous/error/connection.rb
|
166
163
|
- lib/promiscuous/error/dependency.rb
|
167
164
|
- lib/promiscuous/error/lock_unavailable.rb
|
168
165
|
- lib/promiscuous/error/lost_lock.rb
|
169
|
-
- lib/promiscuous/error/missing_context.rb
|
170
166
|
- lib/promiscuous/error/publisher.rb
|
171
167
|
- lib/promiscuous/error/recovery.rb
|
172
168
|
- lib/promiscuous/error/subscriber.rb
|
169
|
+
- lib/promiscuous/error.rb
|
173
170
|
- lib/promiscuous/key.rb
|
174
171
|
- lib/promiscuous/loader.rb
|
175
172
|
- lib/promiscuous/mongoid.rb
|
176
|
-
- lib/promiscuous/publisher.rb
|
177
|
-
- lib/promiscuous/publisher/bootstrap.rb
|
178
|
-
- lib/promiscuous/publisher/bootstrap/connection.rb
|
179
|
-
- lib/promiscuous/publisher/bootstrap/data.rb
|
180
|
-
- lib/promiscuous/publisher/bootstrap/mode.rb
|
181
|
-
- lib/promiscuous/publisher/bootstrap/status.rb
|
182
|
-
- lib/promiscuous/publisher/bootstrap/version.rb
|
183
|
-
- lib/promiscuous/publisher/context.rb
|
184
173
|
- lib/promiscuous/publisher/context/base.rb
|
185
|
-
- lib/promiscuous/publisher/context/middleware.rb
|
186
174
|
- lib/promiscuous/publisher/context/transaction.rb
|
175
|
+
- lib/promiscuous/publisher/context.rb
|
187
176
|
- lib/promiscuous/publisher/mock_generator.rb
|
188
|
-
- lib/promiscuous/publisher/model.rb
|
189
177
|
- lib/promiscuous/publisher/model/active_record.rb
|
190
178
|
- lib/promiscuous/publisher/model/base.rb
|
191
179
|
- lib/promiscuous/publisher/model/ephemeral.rb
|
192
180
|
- lib/promiscuous/publisher/model/mock.rb
|
193
181
|
- lib/promiscuous/publisher/model/mongoid.rb
|
194
|
-
- lib/promiscuous/publisher/
|
182
|
+
- lib/promiscuous/publisher/model.rb
|
195
183
|
- lib/promiscuous/publisher/operation/active_record.rb
|
196
184
|
- lib/promiscuous/publisher/operation/atomic.rb
|
197
185
|
- lib/promiscuous/publisher/operation/base.rb
|
@@ -200,35 +188,30 @@ files:
|
|
200
188
|
- lib/promiscuous/publisher/operation/non_persistent.rb
|
201
189
|
- lib/promiscuous/publisher/operation/proxy_for_query.rb
|
202
190
|
- lib/promiscuous/publisher/operation/transaction.rb
|
191
|
+
- lib/promiscuous/publisher/operation.rb
|
203
192
|
- lib/promiscuous/publisher/worker.rb
|
193
|
+
- lib/promiscuous/publisher.rb
|
204
194
|
- lib/promiscuous/railtie.rb
|
205
195
|
- lib/promiscuous/redis.rb
|
206
|
-
- lib/promiscuous/
|
207
|
-
- lib/promiscuous/sidekiq.rb
|
208
|
-
- lib/promiscuous/subscriber.rb
|
209
|
-
- lib/promiscuous/subscriber/message_processor.rb
|
210
|
-
- lib/promiscuous/subscriber/message_processor/base.rb
|
211
|
-
- lib/promiscuous/subscriber/message_processor/bootstrap.rb
|
212
|
-
- lib/promiscuous/subscriber/message_processor/regular.rb
|
213
|
-
- lib/promiscuous/subscriber/model.rb
|
196
|
+
- lib/promiscuous/subscriber/message.rb
|
214
197
|
- lib/promiscuous/subscriber/model/active_record.rb
|
215
198
|
- lib/promiscuous/subscriber/model/base.rb
|
216
199
|
- lib/promiscuous/subscriber/model/mongoid.rb
|
217
200
|
- lib/promiscuous/subscriber/model/observer.rb
|
201
|
+
- lib/promiscuous/subscriber/model.rb
|
218
202
|
- lib/promiscuous/subscriber/operation.rb
|
219
|
-
- lib/promiscuous/subscriber/
|
220
|
-
- lib/promiscuous/subscriber/operation/bootstrap.rb
|
221
|
-
- lib/promiscuous/subscriber/operation/regular.rb
|
222
|
-
- lib/promiscuous/subscriber/worker.rb
|
203
|
+
- lib/promiscuous/subscriber/unit_of_work.rb
|
223
204
|
- lib/promiscuous/subscriber/worker/eventual_destroyer.rb
|
224
|
-
- lib/promiscuous/subscriber/worker/message.rb
|
225
|
-
- lib/promiscuous/subscriber/worker/message_synchronizer.rb
|
226
205
|
- lib/promiscuous/subscriber/worker/pump.rb
|
227
206
|
- lib/promiscuous/subscriber/worker/recorder.rb
|
228
207
|
- lib/promiscuous/subscriber/worker/runner.rb
|
229
208
|
- lib/promiscuous/subscriber/worker/stats.rb
|
209
|
+
- lib/promiscuous/subscriber/worker.rb
|
210
|
+
- lib/promiscuous/subscriber.rb
|
230
211
|
- lib/promiscuous/timer.rb
|
231
212
|
- lib/promiscuous/version.rb
|
213
|
+
- lib/promiscuous.rb
|
214
|
+
- bin/promiscuous
|
232
215
|
homepage: http://github.com/crowdtap/promiscuous
|
233
216
|
licenses: []
|
234
217
|
metadata: {}
|
@@ -248,9 +231,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
248
231
|
version: '0'
|
249
232
|
requirements: []
|
250
233
|
rubyforge_project:
|
251
|
-
rubygems_version: 2.
|
234
|
+
rubygems_version: 2.0.3
|
252
235
|
signing_key:
|
253
236
|
specification_version: 4
|
254
|
-
summary:
|
237
|
+
summary: Replicate models across applications
|
255
238
|
test_files: []
|
256
|
-
has_rdoc: false
|
@@ -1,29 +0,0 @@
|
|
1
|
-
class Promiscuous::Error::MissingContext < Promiscuous::Error::Base
|
2
|
-
def message
|
3
|
-
require 'erb'
|
4
|
-
ERB.new(<<-ERB.gsub(/^\s+<%/, '<%').gsub(/^ {6}/, ''), nil, '-').result(binding)
|
5
|
-
Promiscuous needs to execute all your read/write queries in a context for publishing.
|
6
|
-
This is what you can do:
|
7
|
-
1. Wrap your operations in a Promiscuous context yourself (jobs, etc.):
|
8
|
-
|
9
|
-
Promiscuous::Middleware.with_context 'jobs/name' do
|
10
|
-
# Code including all your read and write queries
|
11
|
-
end
|
12
|
-
|
13
|
-
2. Disable Promiscuous completely (only for testing):
|
14
|
-
|
15
|
-
RSpec.configure do |config|
|
16
|
-
config.around do |example|
|
17
|
-
without_promiscuous { example.run }
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
Note that opening a context will reactivate promiscuous temporarily
|
22
|
-
even if it was disabled.
|
23
|
-
|
24
|
-
3. You are in render() in the Rails controller, and you should not write.
|
25
|
-
ERB
|
26
|
-
end
|
27
|
-
|
28
|
-
alias to_s message
|
29
|
-
end
|
@@ -1,27 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Bootstrap
|
2
|
-
extend Promiscuous::Autoload
|
3
|
-
autoload :Connection, :Version, :Data, :Mode, :Status
|
4
|
-
|
5
|
-
def self.setup(options={})
|
6
|
-
puts "Enabling bootstrapping mode"
|
7
|
-
Mode.enable
|
8
|
-
puts "Bootstrapping versions..."
|
9
|
-
Version.bootstrap
|
10
|
-
puts "Setting up data bootstrap..."
|
11
|
-
Data.setup(options)
|
12
|
-
end
|
13
|
-
|
14
|
-
def self.run
|
15
|
-
raise "Setup must be run before starting to bootstrap" unless Mode.enabled?
|
16
|
-
Data.run
|
17
|
-
end
|
18
|
-
|
19
|
-
def self.finalize
|
20
|
-
raise "Setup must be run before disabling" unless Mode.enabled?
|
21
|
-
Mode.disable
|
22
|
-
end
|
23
|
-
|
24
|
-
def self.status
|
25
|
-
Status.monitor
|
26
|
-
end
|
27
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::Bootstrap::Connection
|
2
|
-
def initialize
|
3
|
-
# We don't put the connection in the initializer because it gets funny when
|
4
|
-
# it comes to the disconnection.
|
5
|
-
connection_options = { :url => Promiscuous::Config.publisher_amqp_url,
|
6
|
-
:exchange => Promiscuous::AMQP::BOOTSTRAP_EXCHANGE }
|
7
|
-
@connection, @channel, @exchange = Promiscuous::AMQP.new_connection(connection_options)
|
8
|
-
self
|
9
|
-
end
|
10
|
-
|
11
|
-
def publish(options={})
|
12
|
-
options[:key] ||= "#{Promiscuous::Config.app}/__bootstrap__"
|
13
|
-
options[:exchange] ||= @exchange
|
14
|
-
Promiscuous::AMQP.publish(options)
|
15
|
-
end
|
16
|
-
|
17
|
-
def close
|
18
|
-
if @connection
|
19
|
-
# TODO not very pretty... We should abstract that
|
20
|
-
@channel.respond_to?(:stop) ? @channel.stop : @channel.close
|
21
|
-
@connection.respond_to?(:stop) ? @connection.stop : @connection.close
|
22
|
-
@exchange = nil
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,127 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::Bootstrap::Data
|
2
|
-
class << self
|
3
|
-
def setup(options={})
|
4
|
-
Promiscuous::Publisher::Bootstrap::Status.reset
|
5
|
-
range_redis_keys.each { |key| redis.del(key) }
|
6
|
-
|
7
|
-
models = options[:models]
|
8
|
-
models ||= Promiscuous::Publisher::Model.publishers.values
|
9
|
-
.reject { |publisher| publisher.include? Promiscuous::Publisher::Model::Ephemeral }
|
10
|
-
|
11
|
-
models.each_with_index do |model, i|
|
12
|
-
create_range(i, model, options)
|
13
|
-
Promiscuous::Publisher::Bootstrap::Status.total(model.count)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def run(options={})
|
18
|
-
connection = Promiscuous::Publisher::Bootstrap::Connection.new
|
19
|
-
|
20
|
-
range_redis_keys.each do |key|
|
21
|
-
lock = Promiscuous::Redis::Mutex.new(key, lock_options(options[:lock]))
|
22
|
-
if lock.try_lock
|
23
|
-
if range = redis.get(key)
|
24
|
-
range = MultiJson.load(range)
|
25
|
-
selector = range['selector']
|
26
|
-
options = range['options']
|
27
|
-
klass = range['class'].constantize
|
28
|
-
start = range['start'].to_i
|
29
|
-
finish = range['finish'].to_i
|
30
|
-
|
31
|
-
lock_extended_at = Time.now
|
32
|
-
without_promiscuous do
|
33
|
-
range_selector(klass, selector, options, start, finish).each_with_index do |instance, i|
|
34
|
-
publish_data(connection, instance)
|
35
|
-
|
36
|
-
if (Time.now - lock_extended_at) > lock_options[:expire]/5
|
37
|
-
raise "Another worker stole your work!" unless lock.extend
|
38
|
-
lock_extended_at = Time.now
|
39
|
-
end
|
40
|
-
Promiscuous::Publisher::Bootstrap::Status.inc
|
41
|
-
end
|
42
|
-
redis.del(key)
|
43
|
-
lock.unlock
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
private
|
51
|
-
|
52
|
-
def create_range(namespace, model, options)
|
53
|
-
range_size = options[:range_size] || 1000
|
54
|
-
|
55
|
-
without_promiscuous do
|
56
|
-
count = model.all.count
|
57
|
-
return if count == 0
|
58
|
-
|
59
|
-
num_ranges = (count/range_size.to_f).ceil
|
60
|
-
first, last = min_max(model).map { |id| id.to_s.to_i(16) }
|
61
|
-
last += 10 # Ensure that we capture the last ID based on BSON encoding
|
62
|
-
increment = ((last - first)/num_ranges).ceil
|
63
|
-
|
64
|
-
num_ranges.times do |i|
|
65
|
-
range_start = first + (increment * i)
|
66
|
-
range_finish = range_start + increment
|
67
|
-
|
68
|
-
key = range_redis_key.join(namespace).join(i)
|
69
|
-
value = MultiJson.dump(:selector => model.all.selector,
|
70
|
-
:options => model.all.options,
|
71
|
-
:class => model.all.klass.to_s,
|
72
|
-
:start => range_start.to_s,
|
73
|
-
:finish => range_finish.to_s)
|
74
|
-
redis.set(key, value)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
def range_selector(klass, selector, options, start, finish)
|
80
|
-
option ||= {}
|
81
|
-
criteria = Mongoid::Criteria.new(klass)
|
82
|
-
criteria.selector = selector
|
83
|
-
criteria.options = options.merge(:timeout => false)
|
84
|
-
|
85
|
-
criteria.order_by("$natural" => 1).where(:_id => { '$gte' => BSON::ObjectId.from_string(start.to_s(16)),
|
86
|
-
'$lt' => BSON::ObjectId.from_string(finish.to_s(16)) })
|
87
|
-
end
|
88
|
-
|
89
|
-
def range_redis_key
|
90
|
-
Promiscuous::Key.new(:pub).join('bootstrap:range')
|
91
|
-
end
|
92
|
-
|
93
|
-
def range_redis_keys
|
94
|
-
redis.keys("#{range_redis_key}*").reject { |k| k =~ /:lock$/ }.sort
|
95
|
-
end
|
96
|
-
|
97
|
-
def lock_options(options=nil)
|
98
|
-
options ||= {}
|
99
|
-
@@lock_options ||= {
|
100
|
-
:timeout => 10.seconds,
|
101
|
-
:sleep => 0.01.seconds,
|
102
|
-
:expire => 5.minutes,
|
103
|
-
:node => redis
|
104
|
-
}.merge(options)
|
105
|
-
end
|
106
|
-
|
107
|
-
def publish_data(connection, instance)
|
108
|
-
operation = instance.promiscuous.payload
|
109
|
-
operation[:operation] = :bootstrap_data
|
110
|
-
|
111
|
-
payload = {}
|
112
|
-
payload[:app] = Promiscuous::Config.app
|
113
|
-
payload[:operations] = [operation]
|
114
|
-
|
115
|
-
connection.publish(:key => Promiscuous::Config.app, :payload => MultiJson.dump(payload))
|
116
|
-
end
|
117
|
-
|
118
|
-
def min_max(model)
|
119
|
-
query = proc { |sort_order| model.order_by("_id" => sort_order).only(:id).limit(1).first.id }
|
120
|
-
[query.call(1), query.call(-1)]
|
121
|
-
end
|
122
|
-
|
123
|
-
def redis
|
124
|
-
Promiscuous::Redis.master.nodes.first
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Bootstrap::Mode
|
2
|
-
def self.enable
|
3
|
-
Promiscuous::Redis.master.nodes.each { |node| node.set(key, 1) }
|
4
|
-
end
|
5
|
-
|
6
|
-
def self.disable
|
7
|
-
Promiscuous::Redis.master.nodes.each { |node| node.del(key) }
|
8
|
-
end
|
9
|
-
|
10
|
-
def self.enabled?
|
11
|
-
!!Promiscuous::Redis.master.nodes.first.get(key)
|
12
|
-
end
|
13
|
-
|
14
|
-
def self.key
|
15
|
-
# XXX You must change the LUA script in promiscuous/publisher/operation/base.rb
|
16
|
-
# if you change this value
|
17
|
-
Promiscuous::Key.new(:pub).join('bootstrap')
|
18
|
-
end
|
19
|
-
end
|
@@ -1,40 +0,0 @@
|
|
1
|
-
require 'ruby-progressbar'
|
2
|
-
|
3
|
-
module Promiscuous::Publisher::Bootstrap::Status
|
4
|
-
def self.reset
|
5
|
-
redis.del(key)
|
6
|
-
end
|
7
|
-
|
8
|
-
def self.total(count)
|
9
|
-
redis.hincrby(key, 'total', count)
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.inc
|
13
|
-
redis.hincrby(key, 'processed', 1)
|
14
|
-
end
|
15
|
-
|
16
|
-
def self.monitor
|
17
|
-
total ||= redis.hget(key, 'total').to_i
|
18
|
-
processed = 0
|
19
|
-
exit_now = false
|
20
|
-
|
21
|
-
%w(SIGTERM SIGINT).each { |signal| Signal.trap(signal) { exit_now = true } }
|
22
|
-
bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => "Bootstrapping", :total => total)
|
23
|
-
while processed < total
|
24
|
-
processed = redis.hget(key, 'processed').to_i
|
25
|
-
bar.progress = processed
|
26
|
-
sleep 1
|
27
|
-
break if exit_now
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
def self.key
|
34
|
-
Promiscuous::Key.new(:pub).join('bootstrap:counter')
|
35
|
-
end
|
36
|
-
|
37
|
-
def self.redis
|
38
|
-
Promiscuous::Redis.master.nodes.first
|
39
|
-
end
|
40
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::Bootstrap::Version
|
2
|
-
def self.bootstrap
|
3
|
-
connection = Promiscuous::Publisher::Bootstrap::Connection.new
|
4
|
-
Promiscuous::Redis.master.nodes.each_with_index do |node, node_index|
|
5
|
-
begin_at = 0
|
6
|
-
chunk_size = Promiscuous::Config.bootstrap_chunk_size
|
7
|
-
|
8
|
-
while begin_at < Promiscuous::Config.hash_size do
|
9
|
-
end_at = [begin_at + chunk_size, Promiscuous::Config.hash_size].min
|
10
|
-
Chunk.new(connection, node, node_index, (begin_at...end_at)).fetch_and_send
|
11
|
-
begin_at += chunk_size
|
12
|
-
end
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
class Chunk
|
17
|
-
def initialize(connection, node, node_index, range)
|
18
|
-
@connection = connection
|
19
|
-
@node = node
|
20
|
-
@range = range
|
21
|
-
@node_index = node_index
|
22
|
-
end
|
23
|
-
|
24
|
-
def fetch_and_send
|
25
|
-
num_nodes = Promiscuous::Redis.master.nodes.size
|
26
|
-
|
27
|
-
futures = {}
|
28
|
-
@node.pipelined do
|
29
|
-
@range.each do |i|
|
30
|
-
next unless i % num_nodes == @node_index
|
31
|
-
futures[i] = @node.get(Promiscuous::Key.new(:pub).join(i, 'rw'))
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
operation = {}
|
36
|
-
operation[:operation] = :bootstrap_versions
|
37
|
-
operation[:keys] = futures.map { |i, f| "#{i}:#{f.value}" if f.value }.compact
|
38
|
-
|
39
|
-
payload = {}
|
40
|
-
payload[:app] = Promiscuous::Config.app
|
41
|
-
payload[:operations] = [operation]
|
42
|
-
|
43
|
-
@connection.publish(:payload => MultiJson.dump(payload)) if operation[:keys].present?
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|