promiscuous 0.90.0 → 0.91.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/lib/promiscuous/amqp/bunny.rb +63 -36
  3. data/lib/promiscuous/amqp/fake.rb +3 -1
  4. data/lib/promiscuous/amqp/hot_bunnies.rb +26 -16
  5. data/lib/promiscuous/amqp/null.rb +1 -0
  6. data/lib/promiscuous/amqp.rb +12 -12
  7. data/lib/promiscuous/cli.rb +70 -29
  8. data/lib/promiscuous/config.rb +54 -29
  9. data/lib/promiscuous/convenience.rb +1 -1
  10. data/lib/promiscuous/dependency.rb +25 -6
  11. data/lib/promiscuous/error/connection.rb +11 -9
  12. data/lib/promiscuous/error/dependency.rb +8 -1
  13. data/lib/promiscuous/loader.rb +4 -2
  14. data/lib/promiscuous/publisher/bootstrap/connection.rb +25 -0
  15. data/lib/promiscuous/publisher/bootstrap/data.rb +127 -0
  16. data/lib/promiscuous/publisher/bootstrap/mode.rb +19 -0
  17. data/lib/promiscuous/publisher/bootstrap/status.rb +40 -0
  18. data/lib/promiscuous/publisher/bootstrap/version.rb +46 -0
  19. data/lib/promiscuous/publisher/bootstrap.rb +27 -0
  20. data/lib/promiscuous/publisher/context/base.rb +67 -0
  21. data/lib/promiscuous/{middleware.rb → publisher/context/middleware.rb} +16 -13
  22. data/lib/promiscuous/publisher/context/transaction.rb +36 -0
  23. data/lib/promiscuous/publisher/context.rb +4 -88
  24. data/lib/promiscuous/publisher/mock_generator.rb +9 -9
  25. data/lib/promiscuous/publisher/model/active_record.rb +7 -7
  26. data/lib/promiscuous/publisher/model/base.rb +29 -29
  27. data/lib/promiscuous/publisher/model/ephemeral.rb +5 -3
  28. data/lib/promiscuous/publisher/model/mock.rb +9 -5
  29. data/lib/promiscuous/publisher/model/mongoid.rb +5 -22
  30. data/lib/promiscuous/publisher/operation/active_record.rb +360 -0
  31. data/lib/promiscuous/publisher/operation/atomic.rb +167 -0
  32. data/lib/promiscuous/publisher/operation/base.rb +279 -474
  33. data/lib/promiscuous/publisher/operation/mongoid.rb +153 -145
  34. data/lib/promiscuous/publisher/operation/non_persistent.rb +28 -0
  35. data/lib/promiscuous/publisher/operation/proxy_for_query.rb +42 -0
  36. data/lib/promiscuous/publisher/operation/transaction.rb +85 -0
  37. data/lib/promiscuous/publisher/operation.rb +1 -1
  38. data/lib/promiscuous/publisher/worker.rb +7 -7
  39. data/lib/promiscuous/publisher.rb +1 -1
  40. data/lib/promiscuous/railtie.rb +20 -5
  41. data/lib/promiscuous/redis.rb +104 -56
  42. data/lib/promiscuous/subscriber/message_processor/base.rb +38 -0
  43. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +17 -0
  44. data/lib/promiscuous/subscriber/message_processor/regular.rb +192 -0
  45. data/lib/promiscuous/subscriber/message_processor.rb +4 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +20 -15
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +4 -4
  48. data/lib/promiscuous/subscriber/model/observer.rb +16 -2
  49. data/lib/promiscuous/subscriber/operation/base.rb +68 -0
  50. data/lib/promiscuous/subscriber/operation/bootstrap.rb +54 -0
  51. data/lib/promiscuous/subscriber/operation/regular.rb +13 -0
  52. data/lib/promiscuous/subscriber/operation.rb +3 -166
  53. data/lib/promiscuous/subscriber/worker/message.rb +61 -35
  54. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +90 -59
  55. data/lib/promiscuous/subscriber/worker/pump.rb +17 -5
  56. data/lib/promiscuous/subscriber/worker/recorder.rb +4 -1
  57. data/lib/promiscuous/subscriber/worker/runner.rb +49 -9
  58. data/lib/promiscuous/subscriber/worker/stats.rb +2 -2
  59. data/lib/promiscuous/subscriber/worker.rb +6 -0
  60. data/lib/promiscuous/subscriber.rb +1 -1
  61. data/lib/promiscuous/timer.rb +31 -18
  62. data/lib/promiscuous/version.rb +1 -1
  63. data/lib/promiscuous.rb +23 -3
  64. metadata +104 -89
  65. data/lib/promiscuous/subscriber/payload.rb +0 -34
@@ -0,0 +1,167 @@
1
+ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operation::Base
2
+ # XXX instance can be a selector representation.
3
+ attr_accessor :instance
4
+
5
+ def initialize(options={})
6
+ super
7
+ @instance = options[:instance]
8
+ end
9
+
10
+ def acquire_op_lock
11
+ unless dependency_for_op_lock
12
+ return unless reload_instance
13
+ end
14
+
15
+ loop do
16
+ instance_dep = dependency_for_op_lock
17
+
18
+ super
19
+
20
+ return if operation == :create
21
+
22
+ # We need to make sure that the lock we acquired matches our selector.
23
+ # There is a bit of room for optimization if we know that we don't have
24
+ # any tracked attributes on the model and our selector is already an id.
25
+ return unless reload_instance
26
+
27
+ # If reload_instance changed the current instance because the selector,
28
+ # we need to unlock the old instance, lock this new instance, and
29
+ # retry.
30
+ return if instance_dep == dependency_for_op_lock
31
+
32
+ # XXX What should we do if we are going in a live lock?
33
+ # Sleep with some jitter?
34
+ release_op_lock
35
+ end
36
+ end
37
+
38
+ def execute_instrumented(query)
39
+ raise if @instance.nil? # assert()
40
+
41
+ unless self.recovering?
42
+ generate_read_dependencies
43
+ acquire_op_lock
44
+
45
+ if @instance.nil?
46
+ # The selector missed the instance, bailing out.
47
+ query.call_and_remember_result(:non_instrumented)
48
+ return
49
+ end
50
+ end
51
+
52
+ # All the versions are updated and a marked as pending for publish in Redis
53
+ # atomically in case we die before we could write the versions in the
54
+ # database. Once incremented, concurrent queries that are reading our
55
+ # instance will be serialized after our write, even through it may read our
56
+ # old instance. This is a race that we tolerate.
57
+ # XXX We also stash the document for create operations, so the recovery can
58
+ # redo the create to avoid races when instances are getting partitioned.
59
+ increment_read_and_write_dependencies
60
+
61
+ # From this point, if we die, the one expiring our write locks must finish
62
+ # the publish, either by sending a dummy, or by sending the real instance.
63
+ # We could have die before or after the database query.
64
+
65
+ # We save the versions in the database, as it is our source of truth.
66
+ # This allow a reconstruction of redis in the face of failures.
67
+ # We would also need to send a special message to the subscribers to reset
68
+ # their read counters to the last write version since we would not be able
69
+ # to restore the read counters (and we don't want to store them because
70
+ # this would dramatically augment our footprint on the db).
71
+ #
72
+ # If we are doing a destroy operation, and redis dies right after, and
73
+ # we happen to lost contact with rabbitmq, recovery is going to be complex:
74
+ # we would need to do a diff from the dummy subscriber to see what
75
+ # documents are missing on our side to be able to resend the destroy
76
+ # message.
77
+
78
+ case operation
79
+ when :create
80
+ # We don't stash the version in the document as we can't have races
81
+ # on the same document.
82
+ when :update
83
+ stash_version_in_document(@committed_write_deps.first.version)
84
+ # We are now in the possession of an instance that matches the original
85
+ # selector. We need to make sure the db query will operate on it,
86
+ # instead of the original selector.
87
+ use_id_selector(:use_atomic_version_selector => true)
88
+ # We need to use an atomic versioned selector to make sure that
89
+ # if we lose the lock for a long period of time, we don't mess up
90
+ # the record. Perhaps the operation has been recovered a while ago.
91
+ when :destroy
92
+ use_id_selector
93
+ end
94
+
95
+ # The driver is responsible to set instance to the appropriate value.
96
+ query.call_and_remember_result(:instrumented)
97
+
98
+ if query.failed?
99
+ # If we get an network failure, we should retry later.
100
+ return if recoverable_failure?(query.exception)
101
+ @instance = nil
102
+ end
103
+
104
+ # We take a timestamp right after the write is performed because latency
105
+ # measurements are performed on the subscriber.
106
+ record_timestamp
107
+
108
+ # This make sure that if the db operation failed because of a network issue
109
+ # and we got recovered, we don't send anything as we could send a different
110
+ # message than the recovery mechanism.
111
+ ensure_op_still_locked
112
+
113
+ generate_payload
114
+ clear_previous_dependencies
115
+
116
+ # As soon as we unlock the locks, the rescuer will not be able to assume
117
+ # that the database instance is still pristine, and so we need to stash the
118
+ # payload in redis. If redis dies, we don't care because it can be
119
+ # reconstructed. Subscribers can see "compressed" updates.
120
+ publish_payload_in_redis
121
+
122
+ # TODO Performance: merge these 3 redis operations to speed things up.
123
+ release_op_lock
124
+
125
+ # If we die from this point on, a recovery worker can republish our payload
126
+ # since we queued it in Redis.
127
+
128
+ # We don't care if we lost the lock and got recovered, subscribers are
129
+ # immune to duplicate messages.
130
+ publish_payload_in_rabbitmq_async
131
+ end
132
+
133
+ def operation_payloads
134
+ @instance.nil? ? [] : [payload_for(@instance)]
135
+ end
136
+
137
+ def query_dependencies
138
+ dependencies_for(@instance)
139
+ end
140
+
141
+ def fetch_instance
142
+ # This method is overridden to use the original query selector.
143
+ # Should return nil if the instance is not found.
144
+ @instance
145
+ end
146
+
147
+ def reload_instance
148
+ @instance = fetch_instance
149
+ end
150
+
151
+ def stash_version_in_document(version)
152
+ # Overridden to update the query to set the version field with:
153
+ # instance[Promiscuous::Config.version_field] = version
154
+ end
155
+
156
+ def use_id_selector(options={})
157
+ # Overridden to use the {:id => @instance.id} selector.
158
+ # if the option use_atomic_version_selector is passed, the driver must add
159
+ # the version_field selector.
160
+ end
161
+
162
+ def recoverable_failure?(exception)
163
+ # Overridden to tell if the db exception is spurious, like a network
164
+ # failure.
165
+ raise
166
+ end
167
+ end