promiscuous 0.91.0 → 0.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +1 -1
  3. data/lib/promiscuous/amqp.rb +1 -1
  4. data/lib/promiscuous/amqp/fake.rb +0 -3
  5. data/lib/promiscuous/amqp/file.rb +81 -0
  6. data/lib/promiscuous/amqp/null.rb +0 -3
  7. data/lib/promiscuous/cli.rb +35 -25
  8. data/lib/promiscuous/config.rb +8 -3
  9. data/lib/promiscuous/error.rb +1 -2
  10. data/lib/promiscuous/key.rb +1 -12
  11. data/lib/promiscuous/mongoid.rb +7 -0
  12. data/lib/promiscuous/publisher/context/base.rb +4 -4
  13. data/lib/promiscuous/publisher/context/middleware.rb +2 -23
  14. data/lib/promiscuous/publisher/model/ephemeral.rb +5 -1
  15. data/lib/promiscuous/publisher/model/mock.rb +9 -7
  16. data/lib/promiscuous/publisher/model/mongoid.rb +3 -1
  17. data/lib/promiscuous/publisher/operation.rb +1 -1
  18. data/lib/promiscuous/publisher/operation/atomic.rb +44 -32
  19. data/lib/promiscuous/publisher/operation/base.rb +14 -9
  20. data/lib/promiscuous/publisher/operation/ephemeral.rb +14 -0
  21. data/lib/promiscuous/publisher/operation/mongoid.rb +4 -12
  22. data/lib/promiscuous/subscriber/message_processor/base.rb +17 -1
  23. data/lib/promiscuous/subscriber/message_processor/regular.rb +94 -48
  24. data/lib/promiscuous/subscriber/model/active_record.rb +25 -0
  25. data/lib/promiscuous/subscriber/model/base.rb +17 -13
  26. data/lib/promiscuous/subscriber/model/mongoid.rb +20 -1
  27. data/lib/promiscuous/subscriber/model/observer.rb +4 -0
  28. data/lib/promiscuous/subscriber/operation/base.rb +14 -16
  29. data/lib/promiscuous/subscriber/operation/bootstrap.rb +7 -1
  30. data/lib/promiscuous/subscriber/operation/regular.rb +6 -0
  31. data/lib/promiscuous/subscriber/worker.rb +6 -2
  32. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +85 -0
  33. data/lib/promiscuous/subscriber/worker/message.rb +9 -15
  34. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +24 -78
  35. data/lib/promiscuous/subscriber/worker/runner.rb +6 -2
  36. data/lib/promiscuous/subscriber/worker/stats.rb +11 -7
  37. data/lib/promiscuous/version.rb +1 -1
  38. metadata +66 -63
  39. data/lib/promiscuous/error/already_processed.rb +0 -5
@@ -47,5 +47,9 @@ module Promiscuous::Subscriber::Model::Observer
47
47
  new.tap { |o| o.id = id }
48
48
  end
49
49
  alias __promiscuous_fetch_existing __promiscuous_fetch_new
50
+
51
+ def __promiscuous_duplicate_key_exception?(e)
52
+ false
53
+ end
50
54
  end
51
55
  end
@@ -32,33 +32,31 @@ class Promiscuous::Subscriber::Operation::Base
32
32
  instance.save!
33
33
  end
34
34
  rescue Exception => e
35
- # TODO Abstract the duplicated index error message
36
- dup_index_error = true if defined?(Mongoid) && e.message =~ /E11000/
37
- # # TODO Ensure that it's on the pk
38
- dup_index_error = true if defined?(ActiveRecord) && e.is_a?(ActiveRecord::RecordNotUnique)
39
-
40
- if dup_index_error
41
- if options[:upsert]
42
- update
43
- else
44
- warn "ignoring already created record"
45
- end
35
+ if model.__promiscuous_duplicate_key_exception?(e)
36
+ options[:on_already_created] ||= proc { warn "ignoring already created record" }
37
+ options[:on_already_created].call
46
38
  else
47
39
  raise e
48
40
  end
49
41
  end
50
42
 
51
- def update
43
+ def update(should_create_on_failure=true)
52
44
  model.__promiscuous_fetch_existing(id).tap do |instance|
53
- instance.__promiscuous_update(self)
54
- instance.save!
45
+ if instance.__promiscuous_eventual_consistency_update(self)
46
+ instance.__promiscuous_update(self)
47
+ instance.save!
48
+ end
55
49
  end
56
50
  rescue model.__promiscuous_missing_record_exception
57
- warn "upserting #{message.payload}"
58
- create
51
+ warn "upserting"
52
+ create :on_already_created => proc { update(false) if should_create_on_failure }
59
53
  end
60
54
 
61
55
  def destroy
56
+ if Promiscuous::Config.consistency == :eventual
57
+ Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
58
+ end
59
+
62
60
  model.__promiscuous_fetch_existing(id).tap do |instance|
63
61
  instance.destroy
64
62
  end
@@ -14,7 +14,7 @@ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::O
14
14
  end
15
15
 
16
16
  def bootstrap_data
17
- create(:upsert => true)
17
+ create :on_already_created => proc { update }
18
18
  end
19
19
 
20
20
  def on_bootstrap_operation(wanted_operation, options={})
@@ -46,6 +46,12 @@ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::O
46
46
  else
47
47
  raise "Invalid operation received: #{operation}"
48
48
  end
49
+ rescue Exception => e
50
+ if Promiscuous::Config.ignore_exceptions
51
+ Promiscuous.warn "[receive] error while proceessing message but message still processed: #{e}\n#{e.backtrace.join("\n")}"
52
+ else
53
+ raise e
54
+ end
49
55
  end
50
56
 
51
57
  def message_processor
@@ -5,6 +5,12 @@ class Promiscuous::Subscriber::Operation::Regular < Promiscuous::Subscriber::Ope
5
5
  when :update then update if model
6
6
  when :destroy then destroy if model
7
7
  end
8
+ rescue Exception => e
9
+ if Promiscuous::Config.ignore_exceptions && !e.is_a?(NameError)
10
+ Promiscuous.warn "[receive] error while proceessing message but message still processed: #{e}\n#{e.backtrace.join("\n")}"
11
+ else
12
+ raise e
13
+ end
8
14
  end
9
15
 
10
16
  def message_processor
@@ -1,14 +1,16 @@
1
1
  class Promiscuous::Subscriber::Worker
2
2
  extend Promiscuous::Autoload
3
- autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder
3
+ autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder,
4
+ :EventualDestroyer
4
5
 
5
- attr_accessor :message_synchronizer, :pump, :runner, :stats
6
+ attr_accessor :message_synchronizer, :pump, :runner, :stats, :eventual_destroyer
6
7
 
7
8
  def initialize
8
9
  @message_synchronizer = MessageSynchronizer.new(self)
9
10
  @pump = Pump.new(self)
10
11
  @runner = Runner.new(self)
11
12
  @stats = Stats.new
13
+ @eventual_destroyer = EventualDestroyer.new if Promiscuous::Config.consistency == :eventual
12
14
  end
13
15
 
14
16
  def start
@@ -16,6 +18,7 @@ class Promiscuous::Subscriber::Worker
16
18
  @pump.connect
17
19
  @runner.start
18
20
  @stats.connect
21
+ @eventual_destroyer.try(:start)
19
22
  end
20
23
 
21
24
  def stop
@@ -23,6 +26,7 @@ class Promiscuous::Subscriber::Worker
23
26
  @runner.stop
24
27
  @pump.disconnect
25
28
  @message_synchronizer.disconnect
29
+ @eventual_destroyer.try(:stop)
26
30
  end
27
31
 
28
32
  def show_stop_status
@@ -0,0 +1,85 @@
1
+ class Promiscuous::Subscriber::Worker::EventualDestroyer
2
+ def start
3
+ @thread ||= Thread.new { main_loop }
4
+ end
5
+
6
+ def stop
7
+ @thread.try(:kill)
8
+ @thread = nil
9
+ end
10
+
11
+ def self.destroy_timeout
12
+ 1.hour
13
+ end
14
+
15
+ def self.check_every
16
+ (10 + rand(10)).minutes
17
+ end
18
+
19
+ def main_loop
20
+ loop do
21
+ begin
22
+ PendingDestroy.next(self.class.destroy_timeout).each(&:perform)
23
+ rescue Exception => e
24
+ Promiscuous.warn "[eventual destroyer] #{e}\n#{e.backtrace.join("\n")}"
25
+ Promiscuous::Config.error_notifier.call(e)
26
+ end
27
+
28
+ sleep self.class.check_every.to_f
29
+ end
30
+ end
31
+
32
+ def self.postpone_destroy(model, id)
33
+ PendingDestroy.create(:class_name => model.to_s, :instance_id => id)
34
+ end
35
+
36
+ class PendingDestroy
37
+ attr_accessor :class_name, :instance_id
38
+
39
+ def perform
40
+ model = class_name.constantize
41
+ begin
42
+ model.__promiscuous_fetch_existing(instance_id).destroy
43
+ rescue model.__promiscuous_missing_record_exception
44
+ end
45
+
46
+ self.destroy
47
+ end
48
+
49
+ def destroy
50
+ self.class.redis.zrem(self.class.key, @raw)
51
+ end
52
+
53
+ def initialize(raw)
54
+ params = MultiJson.load(raw).with_indifferent_access
55
+
56
+ @class_name = params[:class_name]
57
+ @instance_id = params[:instance_id]
58
+ @raw = raw
59
+ end
60
+
61
+ def self.next(timeout)
62
+ redis.zrangebyscore(key, 0, timeout.ago.utc.to_i).map do |raw|
63
+ self.new(raw)
64
+ end
65
+ end
66
+
67
+ def self.create(options)
68
+ redis.zadd(key, Time.now.utc.to_i, options.to_json)
69
+ end
70
+
71
+ def self.count
72
+ redis.zcard(key)
73
+ end
74
+
75
+ private
76
+
77
+ def self.redis
78
+ Promiscuous::Redis.master.nodes.first
79
+ end
80
+
81
+ def self.key
82
+ Promiscuous::Key.new(:sub).join('eventualdestroyer:jobs')
83
+ end
84
+ end
85
+ end
@@ -23,6 +23,10 @@ class Promiscuous::Subscriber::Worker::Message
23
23
  parsed_payload['timestamp'].to_i
24
24
  end
25
25
 
26
+ def generation
27
+ parsed_payload['generation']
28
+ end
29
+
26
30
  def dependencies
27
31
  @dependencies ||= begin
28
32
  dependencies = parsed_payload['dependencies'] || {}
@@ -53,10 +57,13 @@ class Promiscuous::Subscriber::Worker::Message
53
57
  end
54
58
 
55
59
  def has_dependencies?
56
- return false if Promiscuous::Config.no_deps
57
60
  dependencies.present?
58
61
  end
59
62
 
63
+ def was_during_bootstrap?
64
+ !!parsed_payload['was_during_bootstrap']
65
+ end
66
+
60
67
  def to_s
61
68
  "#{app}/#{context} -> #{happens_before_dependencies.join(', ')}"
62
69
  end
@@ -82,26 +89,13 @@ class Promiscuous::Subscriber::Worker::Message
82
89
  end
83
90
 
84
91
  def unit_of_work(type, &block)
85
- # type is used by the new relic agent, by monkey patching.
86
- # middleware?
87
- if defined?(Mongoid)
88
- Mongoid.unit_of_work { yield_and_catch_already_processsed(&block) }
89
- else
90
- yield_and_catch_already_processsed(&block)
91
- end
92
+ Promiscuous.context { yield }
92
93
  ensure
93
94
  if defined?(ActiveRecord)
94
95
  ActiveRecord::Base.clear_active_connections!
95
96
  end
96
97
  end
97
98
 
98
- def yield_and_catch_already_processsed
99
- Promiscuous.context { yield }
100
- rescue Promiscuous::Error::AlreadyProcessed => orig_e
101
- e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
102
- Promiscuous.debug "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
103
- end
104
-
105
99
  def process
106
100
  unit_of_work(context) do
107
101
  if Promiscuous::Config.bootstrap
@@ -6,7 +6,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
6
6
  CLEANUP_INTERVAL = 100 # messages
7
7
  QUEUE_MAX_AGE = 100 # messages
8
8
 
9
- attr_accessor :redis, :node_synchronizers, :num_processed_messages, :num_queued_messages
9
+ attr_accessor :redis, :node_synchronizers, :num_processed_messages
10
10
 
11
11
  def initialize(root)
12
12
  @root = root
@@ -24,10 +24,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
24
24
  return unless !connected?
25
25
 
26
26
  @num_processed_messages = 0
27
- @num_queued_messages = 0
28
27
  redis = Promiscuous::Redis.new_blocking_connection
29
28
  redis.nodes.each { |node| @node_synchronizers[node] = NodeSynchronizer.new(self, node) }
30
29
  @redis = redis
30
+
31
+ @message_queue = Queue.new
32
+ @processor_thread = Thread.new { queue_process_main_loop }
31
33
  end
32
34
  # Do not recover messages while bootstrapping as there are a very large
33
35
  # number of messages that remain un-acked. If bootstrap messages are missed
@@ -44,6 +46,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
44
46
  @node_synchronizers.values.each { |node_synchronizer| node_synchronizer.stop_main_loop }
45
47
  @node_synchronizers.clear
46
48
  redis.quit
49
+ @processor_thread.kill
47
50
  end
48
51
  rescue Exception
49
52
  end
@@ -70,7 +73,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
70
73
  # If not we bail out and rely on the subscription to kick the processing.
71
74
  # Because we subscribed in advanced, we will not miss the notification.
72
75
  def process_when_ready(msg)
73
- unless msg.has_dependencies?
76
+ unless Promiscuous::Config.consistency == :causal && msg.has_dependencies?
74
77
  process_message!(msg)
75
78
  return
76
79
  end
@@ -78,18 +81,25 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
78
81
  # Dropped messages will be redelivered as we (re)connect
79
82
  return unless self.redis
80
83
 
81
- @lock.synchronize { @num_queued_messages += 1 }
84
+ @message_queue.push([msg, msg.happens_before_dependencies.dup])
85
+ end
86
+
87
+ def queue_process_main_loop
88
+ loop do
89
+ msg, deps = @message_queue.pop
82
90
 
83
- process_message_proc = proc { process_message!(msg) }
84
- msg.happens_before_dependencies.reduce(process_message_proc) do |chain, dep|
85
- get_redis = dep.redis_node
86
- subscriber_redis = dep.redis_node(@redis)
91
+ if dep = deps.pop
92
+ get_redis = dep.redis_node
93
+ subscriber_redis = dep.redis_node(@redis)
87
94
 
88
- key = dep.key(:sub).join('rw').to_s
89
- version = dep.version
90
- node_synchronizer = @node_synchronizers[subscriber_redis]
91
- proc { node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { chain.call } }
92
- end.call
95
+ key = dep.key(:sub).join('rw').to_s
96
+ version = dep.version
97
+ node_synchronizer = @node_synchronizers[subscriber_redis]
98
+ node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { @message_queue.push([msg, deps]) }
99
+ else
100
+ process_message!(msg)
101
+ end
102
+ end
93
103
  end
94
104
 
95
105
  def process_message!(msg)
@@ -99,72 +109,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
99
109
 
100
110
  cleanup = false
101
111
  @lock.synchronize do
102
- @num_queued_messages -= 1
103
112
  @num_processed_messages += 1
104
113
  cleanup = @num_processed_messages % CLEANUP_INTERVAL == 0
105
114
  end
106
115
  @node_synchronizers.values.each(&:cleanup_if_old) if @node_synchronizers && cleanup
107
116
  end
108
117
 
109
- def maybe_recover
110
- if @num_queued_messages == Promiscuous::Config.prefetch
111
- # We've reached the amount of messages the amqp queue is willing to give us.
112
- # We also know that we are not processing messages (@num_queued_messages is
113
- # decremented before we send the message to the runners), and we are called
114
- # after adding a pending callback.
115
- recover_dependencies_for(blocked_messages.first)
116
- end
117
- end
118
-
119
- def recover_dependencies_for(msg)
120
- # XXX This recovery mechanism only works with one worker.
121
- # We are taking the earliest message to unblock, but in reality we should
122
- # do the DAG of the happens before dependencies, take root nodes
123
- # of the disconnected graphs, and sort by timestamps if needed.
124
-
125
- incremented_deps = {}
126
-
127
- msg.happens_before_dependencies.each do |dep|
128
- key = dep.key(:sub).join('rw')
129
- guard_key = key.join('guard') if dep.write?
130
- version = dep.version
131
-
132
- @@version_recovery_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
133
- local key = ARGV[1]
134
- local wanted_version = tonumber(ARGV[2])
135
- local guard_key = ARGV[3]
136
-
137
- if redis.call('exists', guard_key) == 1 then
138
- return
139
- end
140
-
141
- local current_version = tonumber(redis.call('get', key)) or 0
142
-
143
- if wanted_version > current_version then
144
- redis.call('set', guard_key, 1)
145
- redis.call('expire', guard_key, 10)
146
-
147
- redis.call('set', key, wanted_version)
148
- redis.call('publish', key, wanted_version)
149
- return wanted_version - current_version
150
- end
151
- SCRIPT
152
- increment = @@version_recovery_script.eval(dep.redis_node, :argv => [key, version, guard_key].compact)
153
- incremented_deps[dep] = increment if increment
154
- end
155
-
156
- if incremented_deps.present?
157
- recovery_msg = "Incrementing "
158
- recovery_msg += incremented_deps.map { |dep, increment| "#{dep} by #{increment}" }.join(", ")
159
-
160
- e = Promiscuous::Error::Recovery.new(recovery_msg)
161
- Promiscuous.error "[synchronization recovery] #{e}"
162
-
163
- # TODO Should we report the error to the notifier, or the log file is enough?
164
- # Promiscuous::Config.error_notifier.call(e)
165
- end
166
- end
167
-
168
118
  def blocked_messages
169
119
  @node_synchronizers.values
170
120
  .map { |node_synchronizer| node_synchronizer.blocked_messages }
@@ -364,11 +314,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
364
314
  end
365
315
  end
366
316
 
367
- if can_perform_immediately
368
- callback.perform
369
- else
370
- node_synchronizer.root_synchronizer.maybe_recover if Promiscuous::Config.recovery
371
- end
317
+ callback.perform if can_perform_immediately
372
318
  end
373
319
 
374
320
  class Callback < Struct.new(:version, :callback, :message)
@@ -21,7 +21,7 @@ class Promiscuous::Subscriber::Worker::Runner
21
21
  end
22
22
 
23
23
  def show_stop_status(num_requests)
24
- @runner_threads.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
24
+ @runner_threads.to_a.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
25
25
  end
26
26
 
27
27
  class RunnerThread
@@ -43,7 +43,11 @@ class Promiscuous::Subscriber::Worker::Runner
43
43
  end
44
44
 
45
45
  def stop
46
- @kill_lock.synchronize { @thread.kill }
46
+ if @kill_lock.locked? && @thread.stop?
47
+ @thread.kill
48
+ else
49
+ @kill_lock.synchronize { @thread.kill }
50
+ end
47
51
  end
48
52
 
49
53
  def show_stop_status(num_requests)