promiscuous 0.90.0 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/promiscuous/amqp/bunny.rb +63 -36
- data/lib/promiscuous/amqp/fake.rb +3 -1
- data/lib/promiscuous/amqp/hot_bunnies.rb +26 -16
- data/lib/promiscuous/amqp/null.rb +1 -0
- data/lib/promiscuous/amqp.rb +12 -12
- data/lib/promiscuous/cli.rb +70 -29
- data/lib/promiscuous/config.rb +54 -29
- data/lib/promiscuous/convenience.rb +1 -1
- data/lib/promiscuous/dependency.rb +25 -6
- data/lib/promiscuous/error/connection.rb +11 -9
- data/lib/promiscuous/error/dependency.rb +8 -1
- data/lib/promiscuous/loader.rb +4 -2
- data/lib/promiscuous/publisher/bootstrap/connection.rb +25 -0
- data/lib/promiscuous/publisher/bootstrap/data.rb +127 -0
- data/lib/promiscuous/publisher/bootstrap/mode.rb +19 -0
- data/lib/promiscuous/publisher/bootstrap/status.rb +40 -0
- data/lib/promiscuous/publisher/bootstrap/version.rb +46 -0
- data/lib/promiscuous/publisher/bootstrap.rb +27 -0
- data/lib/promiscuous/publisher/context/base.rb +67 -0
- data/lib/promiscuous/{middleware.rb → publisher/context/middleware.rb} +16 -13
- data/lib/promiscuous/publisher/context/transaction.rb +36 -0
- data/lib/promiscuous/publisher/context.rb +4 -88
- data/lib/promiscuous/publisher/mock_generator.rb +9 -9
- data/lib/promiscuous/publisher/model/active_record.rb +7 -7
- data/lib/promiscuous/publisher/model/base.rb +29 -29
- data/lib/promiscuous/publisher/model/ephemeral.rb +5 -3
- data/lib/promiscuous/publisher/model/mock.rb +9 -5
- data/lib/promiscuous/publisher/model/mongoid.rb +5 -22
- data/lib/promiscuous/publisher/operation/active_record.rb +360 -0
- data/lib/promiscuous/publisher/operation/atomic.rb +167 -0
- data/lib/promiscuous/publisher/operation/base.rb +279 -474
- data/lib/promiscuous/publisher/operation/mongoid.rb +153 -145
- data/lib/promiscuous/publisher/operation/non_persistent.rb +28 -0
- data/lib/promiscuous/publisher/operation/proxy_for_query.rb +42 -0
- data/lib/promiscuous/publisher/operation/transaction.rb +85 -0
- data/lib/promiscuous/publisher/operation.rb +1 -1
- data/lib/promiscuous/publisher/worker.rb +7 -7
- data/lib/promiscuous/publisher.rb +1 -1
- data/lib/promiscuous/railtie.rb +20 -5
- data/lib/promiscuous/redis.rb +104 -56
- data/lib/promiscuous/subscriber/message_processor/base.rb +38 -0
- data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +17 -0
- data/lib/promiscuous/subscriber/message_processor/regular.rb +192 -0
- data/lib/promiscuous/subscriber/message_processor.rb +4 -0
- data/lib/promiscuous/subscriber/model/base.rb +20 -15
- data/lib/promiscuous/subscriber/model/mongoid.rb +4 -4
- data/lib/promiscuous/subscriber/model/observer.rb +16 -2
- data/lib/promiscuous/subscriber/operation/base.rb +68 -0
- data/lib/promiscuous/subscriber/operation/bootstrap.rb +54 -0
- data/lib/promiscuous/subscriber/operation/regular.rb +13 -0
- data/lib/promiscuous/subscriber/operation.rb +3 -166
- data/lib/promiscuous/subscriber/worker/message.rb +61 -35
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +90 -59
- data/lib/promiscuous/subscriber/worker/pump.rb +17 -5
- data/lib/promiscuous/subscriber/worker/recorder.rb +4 -1
- data/lib/promiscuous/subscriber/worker/runner.rb +49 -9
- data/lib/promiscuous/subscriber/worker/stats.rb +2 -2
- data/lib/promiscuous/subscriber/worker.rb +6 -0
- data/lib/promiscuous/subscriber.rb +1 -1
- data/lib/promiscuous/timer.rb +31 -18
- data/lib/promiscuous/version.rb +1 -1
- data/lib/promiscuous.rb +23 -3
- metadata +104 -89
- data/lib/promiscuous/subscriber/payload.rb +0 -34
@@ -0,0 +1,167 @@
|
|
1
|
+
class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operation::Base
|
2
|
+
# XXX instance can be a selector representation.
|
3
|
+
attr_accessor :instance
|
4
|
+
|
5
|
+
def initialize(options={})
|
6
|
+
super
|
7
|
+
@instance = options[:instance]
|
8
|
+
end
|
9
|
+
|
10
|
+
def acquire_op_lock
|
11
|
+
unless dependency_for_op_lock
|
12
|
+
return unless reload_instance
|
13
|
+
end
|
14
|
+
|
15
|
+
loop do
|
16
|
+
instance_dep = dependency_for_op_lock
|
17
|
+
|
18
|
+
super
|
19
|
+
|
20
|
+
return if operation == :create
|
21
|
+
|
22
|
+
# We need to make sure that the lock we acquired matches our selector.
|
23
|
+
# There is a bit of room for optimization if we know that we don't have
|
24
|
+
# any tracked attributes on the model and our selector is already an id.
|
25
|
+
return unless reload_instance
|
26
|
+
|
27
|
+
# If reload_instance changed the current instance because the selector,
|
28
|
+
# we need to unlock the old instance, lock this new instance, and
|
29
|
+
# retry.
|
30
|
+
return if instance_dep == dependency_for_op_lock
|
31
|
+
|
32
|
+
# XXX What should we do if we are going in a live lock?
|
33
|
+
# Sleep with some jitter?
|
34
|
+
release_op_lock
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def execute_instrumented(query)
|
39
|
+
raise if @instance.nil? # assert()
|
40
|
+
|
41
|
+
unless self.recovering?
|
42
|
+
generate_read_dependencies
|
43
|
+
acquire_op_lock
|
44
|
+
|
45
|
+
if @instance.nil?
|
46
|
+
# The selector missed the instance, bailing out.
|
47
|
+
query.call_and_remember_result(:non_instrumented)
|
48
|
+
return
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# All the versions are updated and a marked as pending for publish in Redis
|
53
|
+
# atomically in case we die before we could write the versions in the
|
54
|
+
# database. Once incremented, concurrent queries that are reading our
|
55
|
+
# instance will be serialized after our write, even through it may read our
|
56
|
+
# old instance. This is a race that we tolerate.
|
57
|
+
# XXX We also stash the document for create operations, so the recovery can
|
58
|
+
# redo the create to avoid races when instances are getting partitioned.
|
59
|
+
increment_read_and_write_dependencies
|
60
|
+
|
61
|
+
# From this point, if we die, the one expiring our write locks must finish
|
62
|
+
# the publish, either by sending a dummy, or by sending the real instance.
|
63
|
+
# We could have die before or after the database query.
|
64
|
+
|
65
|
+
# We save the versions in the database, as it is our source of truth.
|
66
|
+
# This allow a reconstruction of redis in the face of failures.
|
67
|
+
# We would also need to send a special message to the subscribers to reset
|
68
|
+
# their read counters to the last write version since we would not be able
|
69
|
+
# to restore the read counters (and we don't want to store them because
|
70
|
+
# this would dramatically augment our footprint on the db).
|
71
|
+
#
|
72
|
+
# If we are doing a destroy operation, and redis dies right after, and
|
73
|
+
# we happen to lost contact with rabbitmq, recovery is going to be complex:
|
74
|
+
# we would need to do a diff from the dummy subscriber to see what
|
75
|
+
# documents are missing on our side to be able to resend the destroy
|
76
|
+
# message.
|
77
|
+
|
78
|
+
case operation
|
79
|
+
when :create
|
80
|
+
# We don't stash the version in the document as we can't have races
|
81
|
+
# on the same document.
|
82
|
+
when :update
|
83
|
+
stash_version_in_document(@committed_write_deps.first.version)
|
84
|
+
# We are now in the possession of an instance that matches the original
|
85
|
+
# selector. We need to make sure the db query will operate on it,
|
86
|
+
# instead of the original selector.
|
87
|
+
use_id_selector(:use_atomic_version_selector => true)
|
88
|
+
# We need to use an atomic versioned selector to make sure that
|
89
|
+
# if we lose the lock for a long period of time, we don't mess up
|
90
|
+
# the record. Perhaps the operation has been recovered a while ago.
|
91
|
+
when :destroy
|
92
|
+
use_id_selector
|
93
|
+
end
|
94
|
+
|
95
|
+
# The driver is responsible to set instance to the appropriate value.
|
96
|
+
query.call_and_remember_result(:instrumented)
|
97
|
+
|
98
|
+
if query.failed?
|
99
|
+
# If we get an network failure, we should retry later.
|
100
|
+
return if recoverable_failure?(query.exception)
|
101
|
+
@instance = nil
|
102
|
+
end
|
103
|
+
|
104
|
+
# We take a timestamp right after the write is performed because latency
|
105
|
+
# measurements are performed on the subscriber.
|
106
|
+
record_timestamp
|
107
|
+
|
108
|
+
# This make sure that if the db operation failed because of a network issue
|
109
|
+
# and we got recovered, we don't send anything as we could send a different
|
110
|
+
# message than the recovery mechanism.
|
111
|
+
ensure_op_still_locked
|
112
|
+
|
113
|
+
generate_payload
|
114
|
+
clear_previous_dependencies
|
115
|
+
|
116
|
+
# As soon as we unlock the locks, the rescuer will not be able to assume
|
117
|
+
# that the database instance is still pristine, and so we need to stash the
|
118
|
+
# payload in redis. If redis dies, we don't care because it can be
|
119
|
+
# reconstructed. Subscribers can see "compressed" updates.
|
120
|
+
publish_payload_in_redis
|
121
|
+
|
122
|
+
# TODO Performance: merge these 3 redis operations to speed things up.
|
123
|
+
release_op_lock
|
124
|
+
|
125
|
+
# If we die from this point on, a recovery worker can republish our payload
|
126
|
+
# since we queued it in Redis.
|
127
|
+
|
128
|
+
# We don't care if we lost the lock and got recovered, subscribers are
|
129
|
+
# immune to duplicate messages.
|
130
|
+
publish_payload_in_rabbitmq_async
|
131
|
+
end
|
132
|
+
|
133
|
+
def operation_payloads
|
134
|
+
@instance.nil? ? [] : [payload_for(@instance)]
|
135
|
+
end
|
136
|
+
|
137
|
+
def query_dependencies
|
138
|
+
dependencies_for(@instance)
|
139
|
+
end
|
140
|
+
|
141
|
+
def fetch_instance
|
142
|
+
# This method is overridden to use the original query selector.
|
143
|
+
# Should return nil if the instance is not found.
|
144
|
+
@instance
|
145
|
+
end
|
146
|
+
|
147
|
+
def reload_instance
|
148
|
+
@instance = fetch_instance
|
149
|
+
end
|
150
|
+
|
151
|
+
def stash_version_in_document(version)
|
152
|
+
# Overridden to update the query to set the version field with:
|
153
|
+
# instance[Promiscuous::Config.version_field] = version
|
154
|
+
end
|
155
|
+
|
156
|
+
def use_id_selector(options={})
|
157
|
+
# Overridden to use the {:id => @instance.id} selector.
|
158
|
+
# if the option use_atomic_version_selector is passed, the driver must add
|
159
|
+
# the version_field selector.
|
160
|
+
end
|
161
|
+
|
162
|
+
def recoverable_failure?(exception)
|
163
|
+
# Overridden to tell if the db exception is spurious, like a network
|
164
|
+
# failure.
|
165
|
+
raise
|
166
|
+
end
|
167
|
+
end
|