promiscuous 0.92.0 → 0.100.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +2 -5
  3. data/lib/promiscuous/amqp.rb +1 -2
  4. data/lib/promiscuous/cli.rb +3 -43
  5. data/lib/promiscuous/config.rb +5 -7
  6. data/lib/promiscuous/error/dependency.rb +1 -3
  7. data/lib/promiscuous/publisher/context.rb +1 -1
  8. data/lib/promiscuous/publisher/context/base.rb +3 -34
  9. data/lib/promiscuous/publisher/model/base.rb +5 -25
  10. data/lib/promiscuous/publisher/model/mock.rb +5 -7
  11. data/lib/promiscuous/publisher/operation/active_record.rb +4 -69
  12. data/lib/promiscuous/publisher/operation/atomic.rb +1 -3
  13. data/lib/promiscuous/publisher/operation/base.rb +33 -123
  14. data/lib/promiscuous/publisher/operation/mongoid.rb +0 -67
  15. data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -1
  16. data/lib/promiscuous/publisher/operation/transaction.rb +1 -3
  17. data/lib/promiscuous/railtie.rb +0 -31
  18. data/lib/promiscuous/subscriber.rb +1 -1
  19. data/lib/promiscuous/subscriber/{worker/message.rb → message.rb} +12 -40
  20. data/lib/promiscuous/subscriber/model/active_record.rb +1 -1
  21. data/lib/promiscuous/subscriber/model/base.rb +4 -4
  22. data/lib/promiscuous/subscriber/model/mongoid.rb +3 -3
  23. data/lib/promiscuous/subscriber/operation.rb +74 -3
  24. data/lib/promiscuous/subscriber/unit_of_work.rb +110 -0
  25. data/lib/promiscuous/subscriber/worker.rb +3 -7
  26. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +2 -6
  27. data/lib/promiscuous/subscriber/worker/pump.rb +2 -11
  28. data/lib/promiscuous/version.rb +1 -1
  29. metadata +18 -36
  30. data/lib/promiscuous/error/missing_context.rb +0 -29
  31. data/lib/promiscuous/publisher/bootstrap.rb +0 -27
  32. data/lib/promiscuous/publisher/bootstrap/connection.rb +0 -25
  33. data/lib/promiscuous/publisher/bootstrap/data.rb +0 -127
  34. data/lib/promiscuous/publisher/bootstrap/mode.rb +0 -19
  35. data/lib/promiscuous/publisher/bootstrap/status.rb +0 -40
  36. data/lib/promiscuous/publisher/bootstrap/version.rb +0 -46
  37. data/lib/promiscuous/publisher/context/middleware.rb +0 -94
  38. data/lib/promiscuous/resque.rb +0 -12
  39. data/lib/promiscuous/sidekiq.rb +0 -15
  40. data/lib/promiscuous/subscriber/message_processor.rb +0 -4
  41. data/lib/promiscuous/subscriber/message_processor/base.rb +0 -54
  42. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +0 -17
  43. data/lib/promiscuous/subscriber/message_processor/regular.rb +0 -238
  44. data/lib/promiscuous/subscriber/operation/base.rb +0 -66
  45. data/lib/promiscuous/subscriber/operation/bootstrap.rb +0 -60
  46. data/lib/promiscuous/subscriber/operation/regular.rb +0 -19
  47. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +0 -333
@@ -14,21 +14,12 @@ class Promiscuous::Subscriber::Worker::Pump
14
14
  options[:bindings][exchange] = ['*']
15
15
  end
16
16
 
17
- if Promiscuous::Config.bootstrap
18
- options[:bindings][Promiscuous::AMQP::BOOTSTRAP_EXCHANGE] = ['*']
19
- end
20
-
21
17
  subscribe(options, &method(:on_message))
22
18
  end
23
19
 
24
20
  def on_message(metadata, payload)
25
- msg = Promiscuous::Subscriber::Worker::Message.new(payload, :metadata => metadata, :root_worker => @root)
26
- if Promiscuous::Config.bootstrap
27
- # Bootstrapping doesn't require synchronzation
28
- @root.runner.messages_to_process << msg
29
- else
30
- @root.message_synchronizer.process_when_ready(msg)
31
- end
21
+ msg = Promiscuous::Subscriber::Message.new(payload, :metadata => metadata, :root_worker => @root)
22
+ @root.runner.messages_to_process << msg
32
23
  rescue Exception => e
33
24
  Promiscuous.warn "[receive] cannot process message: #{e}\n#{e.backtrace.join("\n")}"
34
25
  Promiscuous::Config.error_notifier.call(e)
@@ -1,3 +1,3 @@
1
1
  module Promiscuous
2
- VERSION = '0.92.0'
2
+ VERSION = '0.100.0'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: promiscuous
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.92.0
4
+ version: 0.100.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nicolas Viennot
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-02-11 00:00:00.000000000 Z
12
+ date: 2014-04-02 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport
@@ -137,7 +137,7 @@ dependencies:
137
137
  - - ~>
138
138
  - !ruby/object:Gem::Version
139
139
  version: 1.8.0
140
- description: Replicate your Mongoid/ActiveRecord models across your applications
140
+ description: Replicate models across applications
141
141
  email:
142
142
  - nicolas@viennot.biz
143
143
  - kareem@doubleonemedia.com
@@ -146,52 +146,40 @@ executables:
146
146
  extensions: []
147
147
  extra_rdoc_files: []
148
148
  files:
149
- - bin/promiscuous
150
- - lib/promiscuous.rb
151
- - lib/promiscuous/amqp.rb
152
149
  - lib/promiscuous/amqp/bunny.rb
153
150
  - lib/promiscuous/amqp/fake.rb
154
151
  - lib/promiscuous/amqp/file.rb
155
152
  - lib/promiscuous/amqp/hot_bunnies.rb
156
153
  - lib/promiscuous/amqp/null.rb
154
+ - lib/promiscuous/amqp.rb
157
155
  - lib/promiscuous/autoload.rb
158
156
  - lib/promiscuous/cli.rb
159
157
  - lib/promiscuous/config.rb
160
158
  - lib/promiscuous/convenience.rb
161
159
  - lib/promiscuous/dependency.rb
162
160
  - lib/promiscuous/dsl.rb
163
- - lib/promiscuous/error.rb
164
161
  - lib/promiscuous/error/base.rb
165
162
  - lib/promiscuous/error/connection.rb
166
163
  - lib/promiscuous/error/dependency.rb
167
164
  - lib/promiscuous/error/lock_unavailable.rb
168
165
  - lib/promiscuous/error/lost_lock.rb
169
- - lib/promiscuous/error/missing_context.rb
170
166
  - lib/promiscuous/error/publisher.rb
171
167
  - lib/promiscuous/error/recovery.rb
172
168
  - lib/promiscuous/error/subscriber.rb
169
+ - lib/promiscuous/error.rb
173
170
  - lib/promiscuous/key.rb
174
171
  - lib/promiscuous/loader.rb
175
172
  - lib/promiscuous/mongoid.rb
176
- - lib/promiscuous/publisher.rb
177
- - lib/promiscuous/publisher/bootstrap.rb
178
- - lib/promiscuous/publisher/bootstrap/connection.rb
179
- - lib/promiscuous/publisher/bootstrap/data.rb
180
- - lib/promiscuous/publisher/bootstrap/mode.rb
181
- - lib/promiscuous/publisher/bootstrap/status.rb
182
- - lib/promiscuous/publisher/bootstrap/version.rb
183
- - lib/promiscuous/publisher/context.rb
184
173
  - lib/promiscuous/publisher/context/base.rb
185
- - lib/promiscuous/publisher/context/middleware.rb
186
174
  - lib/promiscuous/publisher/context/transaction.rb
175
+ - lib/promiscuous/publisher/context.rb
187
176
  - lib/promiscuous/publisher/mock_generator.rb
188
- - lib/promiscuous/publisher/model.rb
189
177
  - lib/promiscuous/publisher/model/active_record.rb
190
178
  - lib/promiscuous/publisher/model/base.rb
191
179
  - lib/promiscuous/publisher/model/ephemeral.rb
192
180
  - lib/promiscuous/publisher/model/mock.rb
193
181
  - lib/promiscuous/publisher/model/mongoid.rb
194
- - lib/promiscuous/publisher/operation.rb
182
+ - lib/promiscuous/publisher/model.rb
195
183
  - lib/promiscuous/publisher/operation/active_record.rb
196
184
  - lib/promiscuous/publisher/operation/atomic.rb
197
185
  - lib/promiscuous/publisher/operation/base.rb
@@ -200,35 +188,30 @@ files:
200
188
  - lib/promiscuous/publisher/operation/non_persistent.rb
201
189
  - lib/promiscuous/publisher/operation/proxy_for_query.rb
202
190
  - lib/promiscuous/publisher/operation/transaction.rb
191
+ - lib/promiscuous/publisher/operation.rb
203
192
  - lib/promiscuous/publisher/worker.rb
193
+ - lib/promiscuous/publisher.rb
204
194
  - lib/promiscuous/railtie.rb
205
195
  - lib/promiscuous/redis.rb
206
- - lib/promiscuous/resque.rb
207
- - lib/promiscuous/sidekiq.rb
208
- - lib/promiscuous/subscriber.rb
209
- - lib/promiscuous/subscriber/message_processor.rb
210
- - lib/promiscuous/subscriber/message_processor/base.rb
211
- - lib/promiscuous/subscriber/message_processor/bootstrap.rb
212
- - lib/promiscuous/subscriber/message_processor/regular.rb
213
- - lib/promiscuous/subscriber/model.rb
196
+ - lib/promiscuous/subscriber/message.rb
214
197
  - lib/promiscuous/subscriber/model/active_record.rb
215
198
  - lib/promiscuous/subscriber/model/base.rb
216
199
  - lib/promiscuous/subscriber/model/mongoid.rb
217
200
  - lib/promiscuous/subscriber/model/observer.rb
201
+ - lib/promiscuous/subscriber/model.rb
218
202
  - lib/promiscuous/subscriber/operation.rb
219
- - lib/promiscuous/subscriber/operation/base.rb
220
- - lib/promiscuous/subscriber/operation/bootstrap.rb
221
- - lib/promiscuous/subscriber/operation/regular.rb
222
- - lib/promiscuous/subscriber/worker.rb
203
+ - lib/promiscuous/subscriber/unit_of_work.rb
223
204
  - lib/promiscuous/subscriber/worker/eventual_destroyer.rb
224
- - lib/promiscuous/subscriber/worker/message.rb
225
- - lib/promiscuous/subscriber/worker/message_synchronizer.rb
226
205
  - lib/promiscuous/subscriber/worker/pump.rb
227
206
  - lib/promiscuous/subscriber/worker/recorder.rb
228
207
  - lib/promiscuous/subscriber/worker/runner.rb
229
208
  - lib/promiscuous/subscriber/worker/stats.rb
209
+ - lib/promiscuous/subscriber/worker.rb
210
+ - lib/promiscuous/subscriber.rb
230
211
  - lib/promiscuous/timer.rb
231
212
  - lib/promiscuous/version.rb
213
+ - lib/promiscuous.rb
214
+ - bin/promiscuous
232
215
  homepage: http://github.com/crowdtap/promiscuous
233
216
  licenses: []
234
217
  metadata: {}
@@ -248,9 +231,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
248
231
  version: '0'
249
232
  requirements: []
250
233
  rubyforge_project:
251
- rubygems_version: 2.2.2
234
+ rubygems_version: 2.0.3
252
235
  signing_key:
253
236
  specification_version: 4
254
- summary: Model replication over RabbitMQ
237
+ summary: Replicate models across applications
255
238
  test_files: []
256
- has_rdoc: false
@@ -1,29 +0,0 @@
1
- class Promiscuous::Error::MissingContext < Promiscuous::Error::Base
2
- def message
3
- require 'erb'
4
- ERB.new(<<-ERB.gsub(/^\s+<%/, '<%').gsub(/^ {6}/, ''), nil, '-').result(binding)
5
- Promiscuous needs to execute all your read/write queries in a context for publishing.
6
- This is what you can do:
7
- 1. Wrap your operations in a Promiscuous context yourself (jobs, etc.):
8
-
9
- Promiscuous::Middleware.with_context 'jobs/name' do
10
- # Code including all your read and write queries
11
- end
12
-
13
- 2. Disable Promiscuous completely (only for testing):
14
-
15
- RSpec.configure do |config|
16
- config.around do |example|
17
- without_promiscuous { example.run }
18
- end
19
- end
20
-
21
- Note that opening a context will reactivate promiscuous temporarily
22
- even if it was disabled.
23
-
24
- 3. You are in render() in the Rails controller, and you should not write.
25
- ERB
26
- end
27
-
28
- alias to_s message
29
- end
@@ -1,27 +0,0 @@
1
- module Promiscuous::Publisher::Bootstrap
2
- extend Promiscuous::Autoload
3
- autoload :Connection, :Version, :Data, :Mode, :Status
4
-
5
- def self.setup(options={})
6
- puts "Enabling bootstrapping mode"
7
- Mode.enable
8
- puts "Bootstrapping versions..."
9
- Version.bootstrap
10
- puts "Setting up data bootstrap..."
11
- Data.setup(options)
12
- end
13
-
14
- def self.run
15
- raise "Setup must be run before starting to bootstrap" unless Mode.enabled?
16
- Data.run
17
- end
18
-
19
- def self.finalize
20
- raise "Setup must be run before disabling" unless Mode.enabled?
21
- Mode.disable
22
- end
23
-
24
- def self.status
25
- Status.monitor
26
- end
27
- end
@@ -1,25 +0,0 @@
1
- class Promiscuous::Publisher::Bootstrap::Connection
2
- def initialize
3
- # We don't put the connection in the initializer because it gets funny when
4
- # it comes to the disconnection.
5
- connection_options = { :url => Promiscuous::Config.publisher_amqp_url,
6
- :exchange => Promiscuous::AMQP::BOOTSTRAP_EXCHANGE }
7
- @connection, @channel, @exchange = Promiscuous::AMQP.new_connection(connection_options)
8
- self
9
- end
10
-
11
- def publish(options={})
12
- options[:key] ||= "#{Promiscuous::Config.app}/__bootstrap__"
13
- options[:exchange] ||= @exchange
14
- Promiscuous::AMQP.publish(options)
15
- end
16
-
17
- def close
18
- if @connection
19
- # TODO not very pretty... We should abstract that
20
- @channel.respond_to?(:stop) ? @channel.stop : @channel.close
21
- @connection.respond_to?(:stop) ? @connection.stop : @connection.close
22
- @exchange = nil
23
- end
24
- end
25
- end
@@ -1,127 +0,0 @@
1
- class Promiscuous::Publisher::Bootstrap::Data
2
- class << self
3
- def setup(options={})
4
- Promiscuous::Publisher::Bootstrap::Status.reset
5
- range_redis_keys.each { |key| redis.del(key) }
6
-
7
- models = options[:models]
8
- models ||= Promiscuous::Publisher::Model.publishers.values
9
- .reject { |publisher| publisher.include? Promiscuous::Publisher::Model::Ephemeral }
10
-
11
- models.each_with_index do |model, i|
12
- create_range(i, model, options)
13
- Promiscuous::Publisher::Bootstrap::Status.total(model.count)
14
- end
15
- end
16
-
17
- def run(options={})
18
- connection = Promiscuous::Publisher::Bootstrap::Connection.new
19
-
20
- range_redis_keys.each do |key|
21
- lock = Promiscuous::Redis::Mutex.new(key, lock_options(options[:lock]))
22
- if lock.try_lock
23
- if range = redis.get(key)
24
- range = MultiJson.load(range)
25
- selector = range['selector']
26
- options = range['options']
27
- klass = range['class'].constantize
28
- start = range['start'].to_i
29
- finish = range['finish'].to_i
30
-
31
- lock_extended_at = Time.now
32
- without_promiscuous do
33
- range_selector(klass, selector, options, start, finish).each_with_index do |instance, i|
34
- publish_data(connection, instance)
35
-
36
- if (Time.now - lock_extended_at) > lock_options[:expire]/5
37
- raise "Another worker stole your work!" unless lock.extend
38
- lock_extended_at = Time.now
39
- end
40
- Promiscuous::Publisher::Bootstrap::Status.inc
41
- end
42
- redis.del(key)
43
- lock.unlock
44
- end
45
- end
46
- end
47
- end
48
- end
49
-
50
- private
51
-
52
- def create_range(namespace, model, options)
53
- range_size = options[:range_size] || 1000
54
-
55
- without_promiscuous do
56
- count = model.all.count
57
- return if count == 0
58
-
59
- num_ranges = (count/range_size.to_f).ceil
60
- first, last = min_max(model).map { |id| id.to_s.to_i(16) }
61
- last += 10 # Ensure that we capture the last ID based on BSON encoding
62
- increment = ((last - first)/num_ranges).ceil
63
-
64
- num_ranges.times do |i|
65
- range_start = first + (increment * i)
66
- range_finish = range_start + increment
67
-
68
- key = range_redis_key.join(namespace).join(i)
69
- value = MultiJson.dump(:selector => model.all.selector,
70
- :options => model.all.options,
71
- :class => model.all.klass.to_s,
72
- :start => range_start.to_s,
73
- :finish => range_finish.to_s)
74
- redis.set(key, value)
75
- end
76
- end
77
- end
78
-
79
- def range_selector(klass, selector, options, start, finish)
80
- option ||= {}
81
- criteria = Mongoid::Criteria.new(klass)
82
- criteria.selector = selector
83
- criteria.options = options.merge(:timeout => false)
84
-
85
- criteria.order_by("$natural" => 1).where(:_id => { '$gte' => BSON::ObjectId.from_string(start.to_s(16)),
86
- '$lt' => BSON::ObjectId.from_string(finish.to_s(16)) })
87
- end
88
-
89
- def range_redis_key
90
- Promiscuous::Key.new(:pub).join('bootstrap:range')
91
- end
92
-
93
- def range_redis_keys
94
- redis.keys("#{range_redis_key}*").reject { |k| k =~ /:lock$/ }.sort
95
- end
96
-
97
- def lock_options(options=nil)
98
- options ||= {}
99
- @@lock_options ||= {
100
- :timeout => 10.seconds,
101
- :sleep => 0.01.seconds,
102
- :expire => 5.minutes,
103
- :node => redis
104
- }.merge(options)
105
- end
106
-
107
- def publish_data(connection, instance)
108
- operation = instance.promiscuous.payload
109
- operation[:operation] = :bootstrap_data
110
-
111
- payload = {}
112
- payload[:app] = Promiscuous::Config.app
113
- payload[:operations] = [operation]
114
-
115
- connection.publish(:key => Promiscuous::Config.app, :payload => MultiJson.dump(payload))
116
- end
117
-
118
- def min_max(model)
119
- query = proc { |sort_order| model.order_by("_id" => sort_order).only(:id).limit(1).first.id }
120
- [query.call(1), query.call(-1)]
121
- end
122
-
123
- def redis
124
- Promiscuous::Redis.master.nodes.first
125
- end
126
- end
127
- end
@@ -1,19 +0,0 @@
1
- module Promiscuous::Publisher::Bootstrap::Mode
2
- def self.enable
3
- Promiscuous::Redis.master.nodes.each { |node| node.set(key, 1) }
4
- end
5
-
6
- def self.disable
7
- Promiscuous::Redis.master.nodes.each { |node| node.del(key) }
8
- end
9
-
10
- def self.enabled?
11
- !!Promiscuous::Redis.master.nodes.first.get(key)
12
- end
13
-
14
- def self.key
15
- # XXX You must change the LUA script in promiscuous/publisher/operation/base.rb
16
- # if you change this value
17
- Promiscuous::Key.new(:pub).join('bootstrap')
18
- end
19
- end
@@ -1,40 +0,0 @@
1
- require 'ruby-progressbar'
2
-
3
- module Promiscuous::Publisher::Bootstrap::Status
4
- def self.reset
5
- redis.del(key)
6
- end
7
-
8
- def self.total(count)
9
- redis.hincrby(key, 'total', count)
10
- end
11
-
12
- def self.inc
13
- redis.hincrby(key, 'processed', 1)
14
- end
15
-
16
- def self.monitor
17
- total ||= redis.hget(key, 'total').to_i
18
- processed = 0
19
- exit_now = false
20
-
21
- %w(SIGTERM SIGINT).each { |signal| Signal.trap(signal) { exit_now = true } }
22
- bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => "Bootstrapping", :total => total)
23
- while processed < total
24
- processed = redis.hget(key, 'processed').to_i
25
- bar.progress = processed
26
- sleep 1
27
- break if exit_now
28
- end
29
- end
30
-
31
- private
32
-
33
- def self.key
34
- Promiscuous::Key.new(:pub).join('bootstrap:counter')
35
- end
36
-
37
- def self.redis
38
- Promiscuous::Redis.master.nodes.first
39
- end
40
- end
@@ -1,46 +0,0 @@
1
- class Promiscuous::Publisher::Bootstrap::Version
2
- def self.bootstrap
3
- connection = Promiscuous::Publisher::Bootstrap::Connection.new
4
- Promiscuous::Redis.master.nodes.each_with_index do |node, node_index|
5
- begin_at = 0
6
- chunk_size = Promiscuous::Config.bootstrap_chunk_size
7
-
8
- while begin_at < Promiscuous::Config.hash_size do
9
- end_at = [begin_at + chunk_size, Promiscuous::Config.hash_size].min
10
- Chunk.new(connection, node, node_index, (begin_at...end_at)).fetch_and_send
11
- begin_at += chunk_size
12
- end
13
- end
14
- end
15
-
16
- class Chunk
17
- def initialize(connection, node, node_index, range)
18
- @connection = connection
19
- @node = node
20
- @range = range
21
- @node_index = node_index
22
- end
23
-
24
- def fetch_and_send
25
- num_nodes = Promiscuous::Redis.master.nodes.size
26
-
27
- futures = {}
28
- @node.pipelined do
29
- @range.each do |i|
30
- next unless i % num_nodes == @node_index
31
- futures[i] = @node.get(Promiscuous::Key.new(:pub).join(i, 'rw'))
32
- end
33
- end
34
-
35
- operation = {}
36
- operation[:operation] = :bootstrap_versions
37
- operation[:keys] = futures.map { |i, f| "#{i}:#{f.value}" if f.value }.compact
38
-
39
- payload = {}
40
- payload[:app] = Promiscuous::Config.app
41
- payload[:operations] = [operation]
42
-
43
- @connection.publish(:payload => MultiJson.dump(payload)) if operation[:keys].present?
44
- end
45
- end
46
- end