promiscuous 0.100.5 → 1.0.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +5 -1
  3. data/lib/promiscuous/config.rb +6 -5
  4. data/lib/promiscuous/dsl.rb +0 -4
  5. data/lib/promiscuous/loader.rb +0 -5
  6. data/lib/promiscuous/mongoid.rb +15 -5
  7. data/lib/promiscuous/publisher.rb +1 -1
  8. data/lib/promiscuous/publisher/model/active_record.rb +6 -1
  9. data/lib/promiscuous/publisher/model/base.rb +8 -11
  10. data/lib/promiscuous/publisher/model/mock.rb +2 -2
  11. data/lib/promiscuous/publisher/model/mongoid.rb +3 -4
  12. data/lib/promiscuous/publisher/operation/active_record.rb +13 -69
  13. data/lib/promiscuous/publisher/operation/atomic.rb +15 -158
  14. data/lib/promiscuous/publisher/operation/base.rb +13 -381
  15. data/lib/promiscuous/publisher/operation/ephemeral.rb +12 -8
  16. data/lib/promiscuous/publisher/operation/mongoid.rb +22 -92
  17. data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -9
  18. data/lib/promiscuous/publisher/operation/proxy_for_query.rb +8 -6
  19. data/lib/promiscuous/publisher/operation/transaction.rb +4 -56
  20. data/lib/promiscuous/publisher/transport.rb +14 -0
  21. data/lib/promiscuous/publisher/transport/batch.rb +138 -0
  22. data/lib/promiscuous/publisher/transport/persistence.rb +14 -0
  23. data/lib/promiscuous/publisher/transport/persistence/active_record.rb +33 -0
  24. data/lib/promiscuous/publisher/transport/persistence/mongoid.rb +22 -0
  25. data/lib/promiscuous/publisher/transport/worker.rb +36 -0
  26. data/lib/promiscuous/publisher/worker.rb +3 -12
  27. data/lib/promiscuous/redis.rb +5 -0
  28. data/lib/promiscuous/subscriber/message.rb +1 -29
  29. data/lib/promiscuous/subscriber/model/base.rb +3 -2
  30. data/lib/promiscuous/subscriber/model/mongoid.rb +16 -1
  31. data/lib/promiscuous/subscriber/model/observer.rb +0 -1
  32. data/lib/promiscuous/subscriber/operation.rb +9 -3
  33. data/lib/promiscuous/subscriber/unit_of_work.rb +7 -7
  34. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +1 -1
  35. data/lib/promiscuous/version.rb +1 -1
  36. metadata +39 -35
  37. data/lib/promiscuous/dependency.rb +0 -78
  38. data/lib/promiscuous/error/dependency.rb +0 -116
@@ -21,25 +21,6 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
21
21
  rescue NameError
22
22
  end
23
23
 
24
- def recovery_payload
25
- # We use yaml because we need the BSON types.
26
- [@instance.class.promiscuous_collection_name, @instance.id, @document.to_yaml]
27
- end
28
-
29
- def self.recover_operation(collection, instance_id, document)
30
- model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
31
- document = YAML.load(document)
32
- instance = Mongoid::Factory.from_db(model, document)
33
- new(:collection => model.collection, :document => document, :instance => instance)
34
- end
35
-
36
- def recover_db_operation
37
- without_promiscuous do
38
- return if model.unscoped.where(:id => @instance.id).first # already done?
39
- @collection.insert(@document)
40
- end
41
- end
42
-
43
24
  def execute_instrumented(query)
44
25
  @instance = Mongoid::Factory.from_db(model, @document)
45
26
  super
@@ -49,8 +30,8 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
49
30
  super && model
50
31
  end
51
32
 
52
- def recoverable_failure?(exception)
53
- exception.is_a?(Moped::Errors::ConnectionFailure)
33
+ def increment_version_in_document
34
+ @document[Promiscuous::Config.version_field.to_s] = 1
54
35
  end
55
36
  end
56
37
 
@@ -68,8 +49,6 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
68
49
  promiscuous_create_operation(:document => doc).execute { super(doc, flags) }
69
50
  end
70
51
  end
71
-
72
- # TODO aggregate
73
52
  end
74
53
 
75
54
  class Moped::PromiscuousQueryWrapper < Moped::Query
@@ -85,9 +64,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
85
64
  def get_selector_instance
86
65
  selector = @query.operation.selector["$query"] || @query.operation.selector
87
66
 
88
- # TODO use the original instance for an update/delete, that would be
89
- # an even better hint.
90
-
91
67
  # We only support == selectors, no $in, or $gt.
92
68
  @selector = selector.select { |k,v| k.to_s =~ /^[^$]/ && !v.is_a?(Hash) }
93
69
 
@@ -100,6 +76,11 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
100
76
  # reacquired appropriately.
101
77
  model.allocate.tap { |doc| doc.instance_variable_set(:@attributes, @selector) }
102
78
  end
79
+
80
+ def execute_instrumented(query)
81
+ @instance = get_selector_instance
82
+ super
83
+ end
103
84
  end
104
85
 
105
86
  class PromiscuousWriteOperation < Promiscuous::Publisher::Operation::Atomic
@@ -113,61 +94,14 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
113
94
  @change = options[:change]
114
95
  end
115
96
 
116
- def recovery_payload
117
- [@instance.class.promiscuous_collection_name, @instance.id]
118
- end
119
-
120
- def self.recover_operation(collection, instance_id)
121
- # TODO We need to use the primary database. We cannot read from a secondary.
122
- model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
123
- query = model.unscoped.where(:id => instance_id).query
124
-
125
- # We no-op the update operation instead of making it idempotent.
126
- # To do so, we do a dummy update on the document.
127
- # The original caller will fail because the lock was unlocked, so we'll
128
- # won't send a different message.
129
- new(:query => query, :change => {}).tap { |op| op.instance_eval { reload_instance } }
130
- end
131
-
132
- def recover_db_operation
133
- if operation == :update
134
- without_promiscuous { @query.update(@change) }
135
- else
136
- without_promiscuous { @query.remove }
137
- end
138
- end
139
-
140
- def recoverable_failure?(exception)
141
- exception.is_a?(Moped::Errors::ConnectionFailure)
142
- end
143
-
144
97
  def fetch_instance
145
98
  raw_instance = without_promiscuous { @query.first }
146
- Mongoid::Factory.from_db(model, raw_instance) if raw_instance
147
- end
148
-
149
- def use_id_selector(options={})
150
- selector = {'_id' => @instance.id}.merge(@query.selector.select { |k,v| k.to_s.include?("_id") })
151
-
152
- if options[:use_atomic_version_selector]
153
- version = @instance[Promiscuous::Config.version_field]
154
- selector.merge!(Promiscuous::Config.version_field => version)
155
- end
156
-
157
- @query.selector = selector
99
+ @instance = Mongoid::Factory.from_db(model, raw_instance) if raw_instance
158
100
  end
159
101
 
160
102
  def increment_version_in_document
161
103
  @change['$inc'] ||= {}
162
- @change['$inc'][Promiscuous::Config.version_field] = 1
163
- end
164
-
165
- def execute_instrumented(query)
166
- # We are trying to be optimistic for the locking. We are trying to figure
167
- # out our dependencies with the selector upfront to avoid an extra read
168
- # from reload_instance.
169
- @instance ||= get_selector_instance unless recovering? && operation == :update
170
- super
104
+ @change['$inc'][Promiscuous::Config.version_field.to_s] = 1
171
105
  end
172
106
 
173
107
  def fields_in_query(change)
@@ -186,7 +120,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
186
120
  return true unless @change
187
121
 
188
122
  # TODO maybe we should cache these things
189
- # TODO discover field dependencies automatically (hard)
190
123
  aliases = Hash[model.aliased_fields.map { |k,v| [v,k] }]
191
124
  attributes = fields_in_query(@change).map { |f| [aliases[f.to_s], f] }.flatten.compact.map(&:to_sym)
192
125
  (attributes & model.published_db_fields).present?
@@ -206,11 +139,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
206
139
  @query = options[:query]
207
140
  end
208
141
 
209
- def query_dependencies
210
- deps = dependencies_for(get_selector_instance)
211
- deps.empty? ? super : deps
212
- end
213
-
214
142
  def should_instrument_query?
215
143
  super && model
216
144
  end
@@ -236,14 +164,17 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
236
164
 
237
165
  if flags && update_op.should_instrument_query?
238
166
  raise "You cannot do a multi update. Instead, update each document separately." if flags.include?(:multi)
239
- raise "No upsert support yet" if flags.include?(:upsert)
167
+ raise "No upsert support yet" if flags.include?(:upsert) # TODO Should be possible with new architecture
240
168
  end
241
169
 
242
170
  update_op.execute do |query|
243
171
  query.non_instrumented { super }
244
172
  query.instrumented do |op|
245
- raw_instance = without_promiscuous { modify(change, :new => true) }
246
- op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
173
+ if raw_instance = without_promiscuous { modify(change, :new => true) }
174
+ op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
175
+ else
176
+ op.instance = nil
177
+ end
247
178
  {'updatedExisting' => true, 'n' => 1, 'err' => nil, 'ok' => 1.0}
248
179
  end
249
180
  end
@@ -254,7 +185,13 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
254
185
  query.non_instrumented { super }
255
186
  query.instrumented do |op|
256
187
  raise "You can only use find_and_modify() with :new => true" if !options[:new]
257
- super.tap { |raw_instance| op.instance = Mongoid::Factory.from_db(op.model, raw_instance) }
188
+ super.tap do |raw_instance|
189
+ if raw_instance
190
+ op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
191
+ else
192
+ op.instance = nil
193
+ end
194
+ end
258
195
  end
259
196
  end
260
197
  end
@@ -288,13 +225,6 @@ class Moped::PromiscuousDatabase < Moped::Database
288
225
  end
289
226
  end
290
227
 
291
- class Mongoid::Validations::UniquenessValidator
292
- alias_method :validate_root_without_promisucous, :validate_root
293
- def validate_root(*args)
294
- without_promiscuous { validate_root_without_promisucous(*args) }
295
- end
296
- end
297
-
298
228
  Moped.__send__(:remove_const, :Collection)
299
229
  Moped.__send__(:const_set, :Collection, Moped::PromiscuousCollectionWrapper)
300
230
  Moped.__send__(:remove_const, :Query)
@@ -15,13 +15,4 @@ class Promiscuous::Publisher::Operation::NonPersistent < Promiscuous::Publisher:
15
15
  trace_operation
16
16
  end
17
17
  end
18
-
19
- def operation_payloads
20
- return [] if self.failed?
21
- @instances.map { |instance| payloads_for(instance) }
22
- end
23
-
24
- def query_dependencies
25
- @instances.map { |instance| dependencies_for(instance) }
26
- end
27
18
  end
@@ -1,15 +1,17 @@
1
1
  class Promiscuous::Publisher::Operation::ProxyForQuery
2
- attr_accessor :exception, :result
2
+ attr_accessor :exception, :result, :operation
3
3
 
4
4
  def initialize(operation, &block)
5
5
  @operation = operation
6
6
  @queries = {}
7
7
 
8
- if block.arity == 1
9
- block.call(self)
10
- else
11
- self.non_instrumented { block.call }
12
- self.instrumented { block.call }
8
+ if block
9
+ if block.arity == 1
10
+ block.call(self)
11
+ else
12
+ self.non_instrumented { block.call }
13
+ self.instrumented { block.call }
14
+ end
13
15
  end
14
16
  end
15
17
 
@@ -1,5 +1,5 @@
1
1
  class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::Operation::Base
2
- attr_accessor :transaction_id, :transaction_operations, :operation_payloads
2
+ attr_accessor :transaction_id, :transaction_operations
3
3
 
4
4
  def initialize(options={})
5
5
  super
@@ -9,14 +9,6 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
9
9
  @operation_payloads = options[:operation_payloads]
10
10
  end
11
11
 
12
- def dependency_for_op_lock
13
- # We don't take locks on rows as the database already have the locks on them
14
- # until the transaction is committed.
15
- # A lock on the transaction ID is taken so we know when we conflict with
16
- # the recovery mechanism.
17
- Promiscuous::Dependency.new("__transactions__", self.transaction_id, :dont_hash => true)
18
- end
19
-
20
12
  def pending_writes
21
13
  # TODO (performance) Return a list of writes that:
22
14
  # - Never touch the same id (the latest write is sufficient)
@@ -24,60 +16,16 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
24
16
  @transaction_operations
25
17
  end
26
18
 
27
- def query_dependencies
28
- @query_dependencies ||= pending_writes.map(&:query_dependencies).flatten
29
- end
30
-
31
- def operation_payloads
32
- @operation_payloads ||= pending_writes.map(&:operation_payloads).flatten
33
- end
34
-
35
- alias cache_operation_payloads operation_payloads
36
-
37
19
  def should_instrument_query?
38
20
  super && !pending_writes.empty?
39
21
  end
40
22
 
41
23
  def execute_instrumented(query)
42
- unless self.recovering?
43
- acquire_op_lock
44
-
45
- # As opposed to atomic operations, we know the values of the instances
46
- # before the database operation, and not after, so only one stage
47
- # of recovery is used.
48
- cache_operation_payloads
49
-
50
- query.call_and_remember_result(:prepare)
51
- end
52
-
53
- self.increment_dependencies
24
+ transport_batch = create_transport_batch(@transaction_operations)
25
+ transport_batch.prepare
54
26
 
55
27
  query.call_and_remember_result(:instrumented)
56
28
 
57
- # We can't do anything if the prepared commit doesn't go through.
58
- # Either it's a network failure, or the database is having some real
59
- # difficulties. The recovery mechanism will have to retry the transaction.
60
- return if query.failed?
61
-
62
- # We take a timestamp right after the write is performed because latency
63
- # measurements are performed on the subscriber.
64
- record_timestamp
65
-
66
- ensure_op_still_locked
67
-
68
- generate_payload
69
-
70
- publish_payload_in_redis
71
- release_op_lock
72
- publish_payload_in_rabbitmq_async
73
- end
74
-
75
- def recovery_payload
76
- # TODO just save the table/ids, or publish the real payload directly.
77
- [@transaction_id, @operation_payloads]
78
- end
79
-
80
- def self.recover_operation(transaction_id, operation_payloads)
81
- new(:transaction_id => transaction_id, :operation_payloads => operation_payloads)
29
+ transport_batch.publish
82
30
  end
83
31
  end
@@ -0,0 +1,14 @@
1
+ class Promiscuous::Publisher::Transport
2
+ extend Promiscuous::Autoload
3
+ autoload :Batch, :Worker, :Persistence
4
+
5
+ class_attribute :persistence
6
+
7
+ if defined?(Mongoid::Document)
8
+ self.persistence = Persistence::Mongoid.new
9
+ elsif defined?(ActiveRecord::Base)
10
+ self.persistence = Persistence::ActiveRecord.new
11
+ else
12
+ raise "Either Mongoid or ActiveRecord support required"
13
+ end
14
+ end
@@ -0,0 +1,138 @@
1
+ class Promiscuous::Publisher::Transport::Batch
2
+ attr_accessor :operations, :payload_attributes, :timestamp, :id
3
+
4
+ SERIALIZER = MultiJson
5
+
6
+ def self.load(id, dump)
7
+ data = SERIALIZER.load(dump)
8
+
9
+ batch = self.new
10
+ batch.id = id
11
+ batch.timestamp = data['timestamp']
12
+ batch.payload_attributes = data['payload_attributes']
13
+
14
+ data['operations'].each { |op_data| batch.operations << Operation.load(op_data) }
15
+
16
+ batch
17
+ end
18
+
19
+ def initialize
20
+ self.operations = []
21
+ self.payload_attributes = {}
22
+ self.timestamp = Time.now
23
+ end
24
+
25
+ def add(type, instances)
26
+ self.operations << Operation.new(type, instances) if instances.present?
27
+ end
28
+
29
+ def clear
30
+ self.operations = []
31
+ end
32
+
33
+ def prepare
34
+ Promiscuous::Publisher::Transport.persistence.save(self)
35
+ end
36
+
37
+ def publish(raise_error=false)
38
+ Promiscuous::AMQP.ensure_connected
39
+
40
+ begin
41
+ if self.operations.present?
42
+ Promiscuous::AMQP.publish(:key => Promiscuous::Config.app, :payload => self.payload,
43
+ :on_confirm => method(:on_rabbitmq_confirm))
44
+ else
45
+ on_rabbitmq_confirm
46
+ end
47
+ rescue Exception => e
48
+ Promiscuous.warn("[publish] Failure publishing to rabbit #{e}\n#{e.backtrace.join("\n")}")
49
+ raise e if raise_error
50
+ end
51
+ end
52
+
53
+ def on_rabbitmq_confirm
54
+ Promiscuous::Publisher::Transport.persistence.delete(self) if self.id
55
+ end
56
+
57
+ def payload
58
+ payload = {}
59
+ payload[:operations] = self.operations.map(&:payload).flatten
60
+ payload[:app] = Promiscuous::Config.app
61
+ payload[:timestamp] = self.timestamp
62
+ payload[:generation] = Promiscuous::Config.generation
63
+ payload[:host] = Socket.gethostname
64
+
65
+ # Backwards compatibility
66
+ payload[:dependencies] = {}
67
+ payload[:dependencies][:write] = operations.map(&:versions).flatten.map { |v| "xxx:#{v}" }
68
+
69
+ payload.merge!(payload_attributes)
70
+
71
+ MultiJson.dump(payload)
72
+ end
73
+
74
+ def dump
75
+ SERIALIZER.dump(
76
+ {
77
+ :operations => self.operations.map(&:dump),
78
+ :payload_attributes => self.payload_attributes,
79
+ :timestamp => self.timestamp
80
+ })
81
+ end
82
+
83
+ class Operation
84
+ attr_accessor :type, :instances
85
+
86
+ def self.load(dump)
87
+ instances = dump['instances'].map do |attributes|
88
+ if dump['type'] == 'destroy'
89
+ instance_class(attributes).new.tap { |instance| instance.id = attributes['id'] }
90
+ else
91
+ find_instance(attributes)
92
+ end
93
+ end
94
+ self.new(dump['type'], instances)
95
+ end
96
+
97
+ def initialize(type, instances, params={})
98
+ self.type = type
99
+ self.instances = instances
100
+ end
101
+
102
+ def payload
103
+ self.instances.map { |instance| instance.promiscuous.payload(:with_attributes => !destroy?).
104
+ merge(:operation => type, :version => instance.attributes[Promiscuous::Config.version_field]) }
105
+ end
106
+
107
+ def versions
108
+ instances.map { |instance| instance.attributes[Promiscuous::Config.version_field.to_s] }.flatten
109
+ end
110
+
111
+ def dump
112
+ instances_metadata = instances.map do |instance|
113
+ {
114
+ :id => instance.id,
115
+ :class => instance.class.to_s
116
+ }
117
+ end
118
+ {
119
+ :type => type,
120
+ :instances => instances_metadata
121
+ }
122
+ end
123
+
124
+ def destroy?
125
+ type == 'destroy'
126
+ end
127
+
128
+ private
129
+
130
+ def self.find_instance(attributes)
131
+ instance_class(attributes).where(:id => attributes['id']).first
132
+ end
133
+
134
+ def self.instance_class(attributes)
135
+ attributes['class'].constantize
136
+ end
137
+ end
138
+ end