promiscuous 0.92.0 → 0.100.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +2 -5
  3. data/lib/promiscuous/amqp.rb +1 -2
  4. data/lib/promiscuous/cli.rb +3 -43
  5. data/lib/promiscuous/config.rb +5 -7
  6. data/lib/promiscuous/error/dependency.rb +1 -3
  7. data/lib/promiscuous/publisher/context.rb +1 -1
  8. data/lib/promiscuous/publisher/context/base.rb +3 -34
  9. data/lib/promiscuous/publisher/model/base.rb +5 -25
  10. data/lib/promiscuous/publisher/model/mock.rb +5 -7
  11. data/lib/promiscuous/publisher/operation/active_record.rb +4 -69
  12. data/lib/promiscuous/publisher/operation/atomic.rb +1 -3
  13. data/lib/promiscuous/publisher/operation/base.rb +33 -123
  14. data/lib/promiscuous/publisher/operation/mongoid.rb +0 -67
  15. data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -1
  16. data/lib/promiscuous/publisher/operation/transaction.rb +1 -3
  17. data/lib/promiscuous/railtie.rb +0 -31
  18. data/lib/promiscuous/subscriber.rb +1 -1
  19. data/lib/promiscuous/subscriber/{worker/message.rb → message.rb} +12 -40
  20. data/lib/promiscuous/subscriber/model/active_record.rb +1 -1
  21. data/lib/promiscuous/subscriber/model/base.rb +4 -4
  22. data/lib/promiscuous/subscriber/model/mongoid.rb +3 -3
  23. data/lib/promiscuous/subscriber/operation.rb +74 -3
  24. data/lib/promiscuous/subscriber/unit_of_work.rb +110 -0
  25. data/lib/promiscuous/subscriber/worker.rb +3 -7
  26. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +2 -6
  27. data/lib/promiscuous/subscriber/worker/pump.rb +2 -11
  28. data/lib/promiscuous/version.rb +1 -1
  29. metadata +18 -36
  30. data/lib/promiscuous/error/missing_context.rb +0 -29
  31. data/lib/promiscuous/publisher/bootstrap.rb +0 -27
  32. data/lib/promiscuous/publisher/bootstrap/connection.rb +0 -25
  33. data/lib/promiscuous/publisher/bootstrap/data.rb +0 -127
  34. data/lib/promiscuous/publisher/bootstrap/mode.rb +0 -19
  35. data/lib/promiscuous/publisher/bootstrap/status.rb +0 -40
  36. data/lib/promiscuous/publisher/bootstrap/version.rb +0 -46
  37. data/lib/promiscuous/publisher/context/middleware.rb +0 -94
  38. data/lib/promiscuous/resque.rb +0 -12
  39. data/lib/promiscuous/sidekiq.rb +0 -15
  40. data/lib/promiscuous/subscriber/message_processor.rb +0 -4
  41. data/lib/promiscuous/subscriber/message_processor/base.rb +0 -54
  42. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +0 -17
  43. data/lib/promiscuous/subscriber/message_processor/regular.rb +0 -238
  44. data/lib/promiscuous/subscriber/operation/base.rb +0 -66
  45. data/lib/promiscuous/subscriber/operation/bootstrap.rb +0 -60
  46. data/lib/promiscuous/subscriber/operation/regular.rb +0 -19
  47. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +0 -333
@@ -1,94 +0,0 @@
1
- class Promiscuous::Publisher::Context::Middleware < Promiscuous::Publisher::Context::Base
2
- module Controller
3
- extend ActiveSupport::Concern
4
-
5
- def process_action(*args)
6
- full_name = "#{self.class.controller_path}/#{self.action_name}"
7
- current_user = self.current_user if self.respond_to?(:current_user)
8
- Promiscuous::Publisher::Context::Middleware.with_context(full_name, :current_user => current_user) { super }
9
- end
10
- end
11
-
12
- def self.with_context(*args, &block)
13
- # XXX We turn off the disabled flag when entering a middleware.
14
- # It has priority because it's much simpler to use for testing.
15
- old_disabled, Promiscuous.disabled = Promiscuous.disabled?, false
16
- super
17
- rescue Exception => e
18
- $promiscuous_last_exception = e if e.is_a? Promiscuous::Error::Base
19
- pretty_print_exception(e) unless e.is_a? ActionView::MissingTemplate
20
- raise e
21
- ensure
22
- Promiscuous.disabled = old_disabled
23
- end
24
-
25
- def self.pretty_print_exception(e)
26
- return if $promiscuous_pretty_print_exception_once == :disable || ENV['RAILS_ENV'] == 'production'
27
- return if e.is_a?(SystemExit)
28
-
29
- e = e.original_exception if defined?(ActionView::Template::Error) && e.is_a?(ActionView::Template::Error)
30
-
31
- STDERR.puts
32
- STDERR.puts "\e[0;#{36}m/---[ Exception: #{e.class} ]#{'-'*[0, 84 - e.class.name.size].max}\e[0m"
33
- STDERR.puts "\e[0;#{36}m|"
34
-
35
- highlight_indent = false
36
- msg = e.to_s.split("\n").map do |line|
37
- highlight_indent = true if line =~ /The problem comes from the following/ ||
38
- line =~ /Promiscuous is tracking this read/
39
- line = "\e[1;#{31}m#{line}\e[0;#{31}m" if highlight_indent && line =~ /^ /
40
- "\e[0;#{36}m| \e[0;#{31}m#{line}\e[0m"
41
- end
42
-
43
- STDERR.puts msg.join("\n")
44
- STDERR.puts "\e[0;#{36}m|"
45
- STDERR.puts "\e[0;#{36}m+---[ Backtrace ]--------------------------------------------------------------------------------------\e[0m"
46
- STDERR.puts "\e[0;#{36}m|"
47
-
48
- expand = ENV['TRACE'].to_i > 1
49
- bt = e.backtrace.map do |line|
50
- line = case line
51
- when /(rspec-core|instrumentation)/
52
- "\e[1;30m#{line}\e[0m" if expand
53
- when /#{Rails.root}\/app\/controllers/
54
- "\e[1;35m#{line}\e[0m"
55
- when /#{Rails.root}\/app\/models/
56
- "\e[1;33m#{line}\e[0m"
57
- when /#{Rails.root}\/lib/
58
- "\e[1;34m#{line}\e[0m"
59
- when /(mongoid|active_record).*`(count|distinct|each|first|last)'$/
60
- "\e[1;32m#{line}\e[0m"
61
- when /(mongoid|active_record).*`(create|insert|save|update|modify|remove|remove_all)'$/
62
- "\e[1;31m#{line}\e[0m"
63
- when /#{Rails.root}/
64
- if line =~ /\/support\//
65
- "\e[1;30m#{line}\e[0m" if expand
66
- else
67
- "\e[1;36m#{line}\e[0m"
68
- end
69
- else
70
- "\e[1;30m#{line}\e[0m" if expand
71
- end
72
- "\e[0;#{36}m| #{line}" if line
73
- end
74
- .compact
75
- .join("\n")
76
- STDERR.puts bt
77
- STDERR.puts "\e[0;#{36}m|"
78
-
79
- if $cucumber_extra
80
- STDERR.puts "\e[0;#{36}m+---[ Cucumber ]--------------------------------------------------------------------------------------\e[0m"
81
- STDERR.puts "\e[0;#{36}m|"
82
- $cucumber_extra.each_with_index do |line, i|
83
- line = line.gsub(/([^:]*: )(.*)$/, "\\1\e[1;36m\\2")
84
- STDERR.puts "\e[0;#{36}m| \e[0;36m#{line}\e[0m"
85
- STDERR.puts "\e[0;#{36}m|" if i.zero?
86
- end
87
- STDERR.puts "\e[0;#{36}m|"
88
- end
89
-
90
- STDERR.puts "\e[0;#{36}m\\------------------------------------------------------------------------------------------------------\e[0m"
91
- STDERR.puts
92
- $promiscuous_pretty_print_exception_once = :disable if $promiscuous_pretty_print_exception_once
93
- end
94
- end
@@ -1,12 +0,0 @@
1
- require 'resque/job'
2
-
3
- class Resque::Job
4
- alias_method :perform_without_promiscuous, :perform
5
-
6
- def perform
7
- name = "resque/#{payload_class.name.underscore}"
8
- Promiscuous::Middleware.with_context(name) do
9
- perform_without_promiscuous
10
- end
11
- end
12
- end
@@ -1,15 +0,0 @@
1
- module Sidekiq
2
- class Promiscuous
3
- def call(worker_class, item, queue)
4
- ::Promiscuous::Middleware.with_context "sidekiq/#{item['queue']}/#{worker_class.class.to_s.underscore}" do
5
- yield
6
- end
7
- end
8
- end
9
- end
10
-
11
- Sidekiq.configure_server do |config|
12
- config.server_middleware do |chain|
13
- chain.add Sidekiq::Promiscuous
14
- end
15
- end
@@ -1,4 +0,0 @@
1
- module Promiscuous::Subscriber::MessageProcessor
2
- extend Promiscuous::Autoload
3
- autoload :Base, :Regular, :Bootstrap
4
- end
@@ -1,54 +0,0 @@
1
- class Promiscuous::Subscriber::MessageProcessor::Base
2
- attr_accessor :message
3
-
4
- def initialize(message)
5
- self.message = message
6
- end
7
-
8
- def operations
9
- message.parsed_payload['operations'].map { |op| operation_class.new(op) }
10
- end
11
-
12
- def self.process(*args)
13
- raise "Same thread is processing a message?" if self.current
14
-
15
- begin
16
- self.current = new(*args)
17
- self.current.process_message
18
- ensure
19
- self.current = nil
20
- end
21
- end
22
-
23
- def self.current
24
- Thread.current[:promiscuous_message_processor]
25
- end
26
-
27
- def self.current=(value)
28
- Thread.current[:promiscuous_message_processor] = value
29
- end
30
-
31
- def process_message
32
- begin
33
- on_message
34
- rescue Exception => e
35
- @fail_count ||= 0; @fail_count += 1
36
-
37
- if @fail_count <= Promiscuous::Config.max_retries
38
- Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
39
- sleep @fail_count ** 2
40
- process_message
41
- else
42
- raise e
43
- end
44
- end
45
- end
46
-
47
- def on_message
48
- raise "Must be implemented"
49
- end
50
-
51
- def operation_class
52
- raise "Must be implemented"
53
- end
54
- end
@@ -1,17 +0,0 @@
1
- class Promiscuous::Subscriber::MessageProcessor::Bootstrap < Promiscuous::Subscriber::MessageProcessor::Base
2
- def on_message
3
- if bootstrap_operation?
4
- operations.each(&:execute)
5
- else
6
- # Postpone message by doing nothing
7
- end
8
- end
9
-
10
- def bootstrap_operation?
11
- operations.first.try(:operation) =~ /bootstrap/
12
- end
13
-
14
- def operation_class
15
- Promiscuous::Subscriber::Operation::Bootstrap
16
- end
17
- end
@@ -1,238 +0,0 @@
1
- class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscriber::MessageProcessor::Base
2
- delegate :write_dependencies, :read_dependencies, :dependencies, :to => :message
3
-
4
- def nodes_with_deps
5
- @nodes_with_deps ||= dependencies.group_by(&:redis_node)
6
- end
7
-
8
- def instance_dep
9
- @instance_dep ||= write_dependencies.first
10
- end
11
-
12
- def master_node
13
- @master_node ||= instance_dep.redis_node
14
- end
15
-
16
- def master_node_with_deps
17
- @master_node_with_deps ||= nodes_with_deps.select { |node| node == master_node }.first
18
- end
19
-
20
- def secondary_nodes_with_deps
21
- @secondary_nodes_with_deps ||= nodes_with_deps.reject { |node| node == master_node }.to_a
22
- end
23
-
24
- def recovery_key
25
- # We use a recovery_key unique to the operation to avoid any trouble of
26
- # touching another operation.
27
- @recovery_key ||= instance_dep.key(:sub).join(instance_dep.version).to_s
28
- end
29
-
30
- def get_current_instance_version
31
- master_node.get(instance_dep.key(:sub).join('rw').to_s).to_i
32
- end
33
-
34
- # XXX TODO Code is not tolerant to losing a lock.
35
-
36
- def update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options={})
37
- argv = []
38
- argv << MultiJson.dump([r_deps.map { |dep| dep.key(:sub) },
39
- w_deps.map { |dep| dep.key(:sub) },
40
- w_deps.map { |dep| dep.version }])
41
- argv << recovery_key if options[:with_recovery]
42
-
43
- @@update_script_bootstrap ||= Promiscuous::Redis::Script.new <<-SCRIPT
44
- local _args = cjson.decode(ARGV[1])
45
- local read_deps = _args[1]
46
- local write_deps = _args[2]
47
- local write_versions = _args[3]
48
- local recovery_key = ARGV[2]
49
-
50
- if recovery_key and redis.call('exists', recovery_key) == 1 then
51
- return
52
- end
53
-
54
- for i, _key in ipairs(read_deps) do
55
- local key = _key .. ':rw'
56
- local v = redis.call('incr', key)
57
- redis.call('publish', key, v)
58
- end
59
-
60
- for i, _key in ipairs(write_deps) do
61
- local key = _key .. ':rw'
62
- local v = write_versions[i]
63
- local current_version = tonumber(redis.call('get', key)) or 0
64
- if current_version < v then
65
- redis.call('set', key, v)
66
- redis.call('publish', key, v)
67
- end
68
- end
69
-
70
- if recovery_key then
71
- redis.call('set', recovery_key, 'done')
72
- end
73
- SCRIPT
74
-
75
- @@update_script_bootstrap.eval(node, :argv => argv)
76
- end
77
-
78
- def update_dependencies_fast(node, r_deps, w_deps, options={})
79
- keys = (r_deps + w_deps).map { |dep| dep.key(:sub) }
80
- argv = options[:with_recovery] ? [recovery_key] : []
81
-
82
- @@update_script_fast ||= Promiscuous::Redis::Script.new <<-SCRIPT
83
- local deps = KEYS
84
- local recovery_key = ARGV[1]
85
-
86
- if recovery_key and redis.call('exists', recovery_key) == 1 then
87
- return
88
- end
89
-
90
- for i, _key in ipairs(deps) do
91
- local key = _key .. ':rw'
92
- local v = redis.call('incr', key)
93
- redis.call('publish', key, v)
94
- end
95
-
96
- if recovery_key then
97
- redis.call('set', recovery_key, 'done')
98
- end
99
- SCRIPT
100
-
101
- @@update_script_fast.eval(node, :keys => keys, :argv => argv)
102
- end
103
-
104
- def update_dependencies_on_node(node_with_deps, options={})
105
- # Read and write dependencies are not handled the same way:
106
- # * Read dependencies are just incremented (which allow parallelization).
107
- # * Write dependencies are set to be max(current_version, received_version).
108
- # This allow the version bootstrapping process to be non-atomic.
109
- # Publishers upgrade their reads dependencies to write dependencies
110
- # during bootstrapping to permit the mechanism to function properly.
111
-
112
- # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
113
- # worth optimizing it for the non-bootstrap case.
114
-
115
- node = node_with_deps[0]
116
- r_deps = node_with_deps[1].select(&:read?)
117
- w_deps = node_with_deps[1].select(&:write?)
118
-
119
- if message.was_during_bootstrap?
120
- raise "Message should not have any read deps" unless r_deps.empty?
121
- update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options)
122
- else
123
- update_dependencies_fast(node, r_deps, w_deps, options)
124
- end
125
- end
126
-
127
- def update_dependencies_master(options={})
128
- update_dependencies_on_node(master_node_with_deps, options)
129
- end
130
-
131
- def update_dependencies_secondaries(options={})
132
- secondary_nodes_with_deps.each do |node_with_deps|
133
- update_dependencies_on_node(node_with_deps, options.merge(:with_recovery => true))
134
- after_secondary_update_hook
135
- end
136
- end
137
-
138
- def after_secondary_update_hook
139
- # Hook only used for testing
140
- end
141
-
142
- def cleanup_dependency_secondaries
143
- secondary_nodes_with_deps.each do |node, deps|
144
- node.del(recovery_key)
145
- end
146
- end
147
-
148
- def update_dependencies(options={})
149
- # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
150
- # 1) We do the secondaries first, with a recovery token.
151
- # 2) Then we do the master.
152
- # 3) Then we cleanup the recovery token on secondaries.
153
- update_dependencies_secondaries(options)
154
- update_dependencies_master(options)
155
- cleanup_dependency_secondaries
156
- end
157
-
158
- def duplicate_message?
159
- unless instance_dep.version >= get_current_instance_version + 1
160
- # We happen to get a duplicate message, or we are recovering a dead
161
- # worker. During regular operations, we just need to cleanup the 2pc (from
162
- # the dead worker), and ack the message to rabbit.
163
- # TODO Test cleanup
164
- cleanup_dependency_secondaries
165
-
166
- # But, if the message was generated during bootstrap, we don't really know
167
- # if the other dependencies are up to date (because of the non-atomic
168
- # bootstrapping process), so we do the max() trick (see in update_dependencies_on_node).
169
- # Since such messages can come arbitrary late, we never really know if we
170
- # can assume regular operations, thus we always assume that such message
171
- # can originate from the bootstrapping period.
172
- # Note that we are not in the happy path. Such duplicates messages are
173
- # seldom: either (1) the publisher recovered a payload that didn't need
174
- # recovery, or (2) a subscriber worker died after # update_dependencies_master,
175
- # but before the message acking).
176
- update_dependencies if message.was_during_bootstrap?
177
-
178
- true
179
- else
180
- false
181
- end
182
- end
183
-
184
- LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute, we give up
185
- :sleep => 0.1, # polling every 100ms.
186
- :expire => 1.minute } # after one minute, we are considered dead
187
-
188
- def check_duplicate_and_update_dependencies
189
- if duplicate_message?
190
- Promiscuous.debug "[receive] Skipping message (already processed) #{message}"
191
- return
192
- end
193
-
194
- yield
195
-
196
- update_dependencies
197
- end
198
-
199
- def with_instance_locked(&block)
200
- return yield unless message.has_dependencies?
201
-
202
- lock_options = LOCK_OPTIONS.merge(:node => master_node)
203
- mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
204
-
205
- unless mutex.lock
206
- raise Promiscuous::Error::LockUnavailable.new(mutex.key)
207
- end
208
-
209
- begin
210
- yield
211
- ensure
212
- unless mutex.unlock
213
- # TODO Be safe in case we have a duplicate message and lost the lock on it
214
- raise "The subscriber lost the lock during its operation. It means that someone else\n"+
215
- "received a duplicate message, and we got screwed.\n"
216
- end
217
- end
218
- end
219
-
220
- def execute_operations
221
- self.operations.each(&:execute)
222
- end
223
-
224
- def on_message
225
- with_instance_locked do
226
- if Promiscuous::Config.consistency == :causal && message.has_dependencies?
227
- self.check_duplicate_and_update_dependencies { execute_operations }
228
- else
229
- execute_operations
230
- end
231
- end
232
- message.ack
233
- end
234
-
235
- def operation_class
236
- Promiscuous::Subscriber::Operation::Regular
237
- end
238
- end
@@ -1,66 +0,0 @@
1
- class Promiscuous::Subscriber::Operation::Base
2
- attr_accessor :model, :id, :operation, :attributes
3
- delegate :message, :to => :message_processor
4
-
5
- def initialize(payload)
6
- if payload.is_a?(Hash)
7
- self.id = payload['id']
8
- self.operation = payload['operation'].try(:to_sym)
9
- self.attributes = payload['attributes']
10
- self.model = self.get_subscribed_model(payload) if payload['types']
11
- end
12
- end
13
-
14
- def get_subscribed_model(payload)
15
- [message.app, '*'].each do |app|
16
- app_mapping = Promiscuous::Subscriber::Model.mapping[app] || {}
17
- payload['types'].to_a.each do |ancestor|
18
- model = app_mapping[ancestor]
19
- return model if model
20
- end
21
- end
22
- nil
23
- end
24
-
25
- def warn(msg)
26
- Promiscuous.warn "[receive] #{msg} #{message.payload}"
27
- end
28
-
29
- def create(options={})
30
- model.__promiscuous_fetch_new(id).tap do |instance|
31
- instance.__promiscuous_update(self)
32
- instance.save!
33
- end
34
- rescue Exception => e
35
- if model.__promiscuous_duplicate_key_exception?(e)
36
- options[:on_already_created] ||= proc { warn "ignoring already created record" }
37
- options[:on_already_created].call
38
- else
39
- raise e
40
- end
41
- end
42
-
43
- def update(should_create_on_failure=true)
44
- model.__promiscuous_fetch_existing(id).tap do |instance|
45
- if instance.__promiscuous_eventual_consistency_update(self)
46
- instance.__promiscuous_update(self)
47
- instance.save!
48
- end
49
- end
50
- rescue model.__promiscuous_missing_record_exception
51
- warn "upserting"
52
- create :on_already_created => proc { update(false) if should_create_on_failure }
53
- end
54
-
55
- def destroy
56
- if Promiscuous::Config.consistency == :eventual
57
- Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
58
- end
59
-
60
- model.__promiscuous_fetch_existing(id).tap do |instance|
61
- instance.destroy
62
- end
63
- rescue model.__promiscuous_missing_record_exception
64
- warn "ignoring missing record"
65
- end
66
- end