promiscuous 0.53.1 → 0.90.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/lib/promiscuous.rb +25 -28
  2. data/lib/promiscuous/amqp.rb +27 -8
  3. data/lib/promiscuous/amqp/bunny.rb +131 -16
  4. data/lib/promiscuous/amqp/fake.rb +52 -0
  5. data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
  6. data/lib/promiscuous/amqp/null.rb +6 -6
  7. data/lib/promiscuous/cli.rb +108 -24
  8. data/lib/promiscuous/config.rb +73 -12
  9. data/lib/promiscuous/convenience.rb +18 -0
  10. data/lib/promiscuous/dependency.rb +59 -0
  11. data/lib/promiscuous/dsl.rb +36 -0
  12. data/lib/promiscuous/error.rb +3 -1
  13. data/lib/promiscuous/error/already_processed.rb +5 -0
  14. data/lib/promiscuous/error/base.rb +1 -0
  15. data/lib/promiscuous/error/connection.rb +7 -5
  16. data/lib/promiscuous/error/dependency.rb +111 -0
  17. data/lib/promiscuous/error/lock_unavailable.rb +12 -0
  18. data/lib/promiscuous/error/lost_lock.rb +12 -0
  19. data/lib/promiscuous/error/missing_context.rb +29 -0
  20. data/lib/promiscuous/error/publisher.rb +5 -15
  21. data/lib/promiscuous/error/recovery.rb +7 -0
  22. data/lib/promiscuous/error/subscriber.rb +2 -4
  23. data/lib/promiscuous/key.rb +36 -0
  24. data/lib/promiscuous/loader.rb +12 -16
  25. data/lib/promiscuous/middleware.rb +112 -0
  26. data/lib/promiscuous/publisher.rb +7 -4
  27. data/lib/promiscuous/publisher/context.rb +92 -0
  28. data/lib/promiscuous/publisher/mock_generator.rb +72 -0
  29. data/lib/promiscuous/publisher/model.rb +3 -86
  30. data/lib/promiscuous/publisher/model/active_record.rb +8 -15
  31. data/lib/promiscuous/publisher/model/base.rb +136 -0
  32. data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
  33. data/lib/promiscuous/publisher/model/mock.rb +61 -0
  34. data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
  35. data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
  36. data/lib/promiscuous/publisher/operation/base.rb +707 -0
  37. data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
  38. data/lib/promiscuous/publisher/worker.rb +22 -0
  39. data/lib/promiscuous/railtie.rb +21 -3
  40. data/lib/promiscuous/redis.rb +132 -40
  41. data/lib/promiscuous/resque.rb +12 -0
  42. data/lib/promiscuous/sidekiq.rb +15 -0
  43. data/lib/promiscuous/subscriber.rb +9 -20
  44. data/lib/promiscuous/subscriber/model.rb +4 -104
  45. data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +96 -0
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
  48. data/lib/promiscuous/subscriber/model/observer.rb +37 -0
  49. data/lib/promiscuous/subscriber/operation.rb +167 -0
  50. data/lib/promiscuous/subscriber/payload.rb +34 -0
  51. data/lib/promiscuous/subscriber/worker.rb +22 -18
  52. data/lib/promiscuous/subscriber/worker/message.rb +48 -25
  53. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
  54. data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
  55. data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
  56. data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
  57. data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
  58. data/lib/promiscuous/timer.rb +38 -0
  59. data/lib/promiscuous/version.rb +1 -1
  60. metadata +98 -143
  61. data/README.md +0 -33
  62. data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
  63. data/lib/promiscuous/common.rb +0 -4
  64. data/lib/promiscuous/common/class_helpers.rb +0 -12
  65. data/lib/promiscuous/common/lint/base.rb +0 -24
  66. data/lib/promiscuous/common/options.rb +0 -51
  67. data/lib/promiscuous/ephemeral.rb +0 -14
  68. data/lib/promiscuous/error/recover.rb +0 -1
  69. data/lib/promiscuous/observer.rb +0 -5
  70. data/lib/promiscuous/publisher/active_record.rb +0 -7
  71. data/lib/promiscuous/publisher/amqp.rb +0 -18
  72. data/lib/promiscuous/publisher/attributes.rb +0 -32
  73. data/lib/promiscuous/publisher/base.rb +0 -23
  74. data/lib/promiscuous/publisher/class.rb +0 -36
  75. data/lib/promiscuous/publisher/envelope.rb +0 -7
  76. data/lib/promiscuous/publisher/ephemeral.rb +0 -9
  77. data/lib/promiscuous/publisher/lint.rb +0 -35
  78. data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
  79. data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
  80. data/lib/promiscuous/publisher/lint/base.rb +0 -5
  81. data/lib/promiscuous/publisher/lint/class.rb +0 -15
  82. data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
  83. data/lib/promiscuous/publisher/mock.rb +0 -79
  84. data/lib/promiscuous/publisher/mongoid.rb +0 -33
  85. data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
  86. data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
  87. data/lib/promiscuous/publisher/polymorphic.rb +0 -8
  88. data/lib/promiscuous/subscriber/active_record.rb +0 -11
  89. data/lib/promiscuous/subscriber/amqp.rb +0 -25
  90. data/lib/promiscuous/subscriber/attributes.rb +0 -35
  91. data/lib/promiscuous/subscriber/base.rb +0 -29
  92. data/lib/promiscuous/subscriber/class.rb +0 -29
  93. data/lib/promiscuous/subscriber/dummy.rb +0 -19
  94. data/lib/promiscuous/subscriber/envelope.rb +0 -18
  95. data/lib/promiscuous/subscriber/lint.rb +0 -30
  96. data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
  97. data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
  98. data/lib/promiscuous/subscriber/lint/base.rb +0 -14
  99. data/lib/promiscuous/subscriber/lint/class.rb +0 -13
  100. data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
  101. data/lib/promiscuous/subscriber/mongoid.rb +0 -27
  102. data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
  103. data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
  104. data/lib/promiscuous/subscriber/observer.rb +0 -26
  105. data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
  106. data/lib/promiscuous/subscriber/upsert.rb +0 -12
@@ -1,30 +0,0 @@
1
- module Promiscuous::Subscriber::Lint
2
- extend Promiscuous::Autoload
3
- autoload :Base, :Class, :Attributes, :Polymorphic, :AMQP
4
-
5
- def self.lint(binding_classes={})
6
- Base.reload_publishers
7
-
8
- if binding_classes.empty?
9
- binding_classes = Promiscuous::Subscriber::AMQP.subscribers.reduce({}) do |res, e|
10
- from, sub = e
11
- res[from] = sub.klass unless from =~ /^__promiscuous__/
12
- res
13
- end
14
- raise "No subscribers found" if binding_classes.empty?
15
- end
16
-
17
- binding_classes.each do |from, klass|
18
- sub = Promiscuous::Subscriber::AMQP.subscribers[from]
19
- raise "#{from} has no subscriber" if sub.nil?
20
-
21
- lint = ::Class.new(Base)
22
- lint.__send__(:include, Class) if sub.include?(Promiscuous::Subscriber::Class)
23
- lint.__send__(:include, Attributes) if sub.include?(Promiscuous::Subscriber::Attributes)
24
- lint.__send__(:include, AMQP) if sub.include?(Promiscuous::Subscriber::AMQP)
25
- lint.__send__(:include, Polymorphic) if sub.include?(Promiscuous::Subscriber::Polymorphic)
26
- lint.new(:klass => klass, :subscriber => sub, :from => from).lint
27
- end
28
- true
29
- end
30
- end
@@ -1,21 +0,0 @@
1
- module Promiscuous::Subscriber::Lint::AMQP
2
- extend ActiveSupport::Concern
3
-
4
- def publisher
5
- publishers.
6
- select { |pub| pub.superclass == Promiscuous::Publisher::Mock }.
7
- select { |pub| pub.to == from }.
8
- tap { |pubs| raise "#{from} has multiple publishers: #{pubs}" if pubs.size > 1 }.
9
- first
10
- end
11
-
12
- def lint
13
- super
14
-
15
- if check_publisher
16
- raise "No publisher found for #{publisher}" if publisher.nil?
17
- end
18
- end
19
-
20
- included { use_option :from }
21
- end
@@ -1,21 +0,0 @@
1
- module Promiscuous::Subscriber::Lint::Attributes
2
- extend ActiveSupport::Concern
3
-
4
- def lint
5
- super
6
-
7
- instance = subscriber.klass.new
8
- attributes = subscriber.attributes
9
- attributes += [subscriber.foreign_key] if subscriber.foreign_key
10
-
11
- attributes.each { |attr| instance.respond_to?("#{attr}=") or instance.__send__("#{attr}=") }
12
-
13
- if check_publisher
14
- raise "The publisher of #{subscriber} does not exist" if publisher.nil?
15
- missing_attributes = subscriber.attributes - publisher.attributes
16
- if missing_attributes.present?
17
- raise "#{publisher} subscribes to non published attributes: #{missing_attributes.join(", ")}"
18
- end
19
- end
20
- end
21
- end
@@ -1,14 +0,0 @@
1
- class Promiscuous::Subscriber::Lint::Base
2
- include Promiscuous::Common::Lint::Base
3
-
4
- class_attribute :publishers
5
- def self.reload_publishers
6
- self.publishers = Promiscuous::Publisher::Mock.descendants
7
- end
8
-
9
- def check_publisher
10
- self.class.publishers.present?
11
- end
12
-
13
- use_option(:subscriber)
14
- end
@@ -1,13 +0,0 @@
1
- module Promiscuous::Subscriber::Lint::Class
2
- extend ActiveSupport::Concern
3
-
4
- def lint
5
- super
6
-
7
- if klass && subscriber.klass != klass
8
- raise "Subscriber #{subscriber} does not replicate #{klass}"
9
- end
10
- end
11
-
12
- included { use_option :klass }
13
- end
@@ -1,39 +0,0 @@
1
- module Promiscuous::Subscriber::Lint::Polymorphic
2
- extend ActiveSupport::Concern
3
-
4
- def publisher
5
- parent_publisher = super
6
- return nil if parent_publisher.nil?
7
-
8
- publishers.
9
- select { |pub| pub <= parent_publisher }.
10
- select { |pub| pub.class_name == subscriber.from_type }.
11
- first
12
- end
13
-
14
- def lint
15
- super
16
- return if skip_polymorphic
17
-
18
- sub_descendants = subscriber.descendants
19
- pub_descendants = publishers.select { |pub| pub < publisher }
20
-
21
- if check_publisher
22
- subscriber_types = sub_descendants.map &:from_type
23
- publisher_types = pub_descendants.map &:class_name
24
- missing_types = publisher_types - subscriber_types
25
- if missing_types.present?
26
- raise "#{subscriber} does not cover #{missing_types.join(", ")}"
27
- end
28
- end
29
-
30
- sub_descendants.each do |pub|
31
- self.class.new(options.merge(:klass => nil, :subscriber => pub,
32
- :skip_polymorphic => true)).lint
33
- end
34
- end
35
-
36
- included do
37
- use_option :skip_polymorphic
38
- end
39
- end
@@ -1,27 +0,0 @@
1
- class Promiscuous::Subscriber::Mongoid < Promiscuous::Subscriber::Base
2
- extend Promiscuous::Autoload
3
- autoload :Embedded
4
-
5
- include Promiscuous::Subscriber::Class
6
- include Promiscuous::Subscriber::Attributes
7
- include Promiscuous::Subscriber::Polymorphic
8
- include Promiscuous::Subscriber::AMQP
9
-
10
- def self.missing_record_exception
11
- Mongoid::Errors::DocumentNotFound
12
- end
13
-
14
- def self.subscribe(options)
15
- super
16
-
17
- if klass.embedded?
18
- require 'promiscuous/subscriber/mongoid/embedded_many'
19
- include Promiscuous::Subscriber::Mongoid::Embedded
20
- else
21
- include Promiscuous::Subscriber::Model
22
- include Promiscuous::Subscriber::Upsert
23
- end
24
-
25
- setup_class_binding
26
- end
27
- end
@@ -1,17 +0,0 @@
1
- module Promiscuous::Subscriber::Mongoid::Embedded
2
- extend ActiveSupport::Concern
3
-
4
- def fetch
5
- (old_value || klass.new).tap { |m| m.id = id }
6
- end
7
-
8
- def should_update_parent?
9
- old_value.nil?
10
- end
11
-
12
- def old_value
13
- options[:old_value]
14
- end
15
-
16
- included { use_payload_attribute :id }
17
- end
@@ -1,44 +0,0 @@
1
- class Promiscuous::Subscriber::Mongoid::EmbeddedMany < Promiscuous::Subscriber::Base
2
- include Promiscuous::Subscriber::AMQP
3
-
4
- subscribe :from => '__promiscuous__/embedded_many'
5
-
6
- def old_embeddeds
7
- options[:old_value]
8
- end
9
-
10
- def parent
11
- options[:parent]
12
- end
13
-
14
- alias :new_embeddeds :payload
15
-
16
- def process
17
- # XXX Reordering is not supported
18
-
19
- # find all updatable docs
20
- new_embeddeds.each do |new_e|
21
- old_e = old_embeddeds.select { |e| e.id.to_s == new_e['id'] }.first
22
- if old_e
23
- new_e['existed'] = true
24
- old_e.instance_variable_set(:@keep, true)
25
- Promiscuous::Subscriber.process(new_e, :old_value => old_e)
26
- end
27
- end
28
-
29
- # delete all the old ones
30
- old_embeddeds.each do |old_e|
31
- old_e.destroy unless old_e.instance_variable_get(:@keep)
32
- end
33
-
34
- # create all the new ones
35
- new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
36
- new_e_instance = Promiscuous::Subscriber.process(new_e)
37
- parent.__send__(old_embeddeds.metadata[:name]) << new_e_instance
38
- end
39
- end
40
-
41
- def should_update_parent?
42
- false
43
- end
44
- end
@@ -1,26 +0,0 @@
1
- class Promiscuous::Subscriber::Observer < Promiscuous::Subscriber::Base
2
- include Promiscuous::Subscriber::Class
3
- include Promiscuous::Subscriber::Attributes
4
- include Promiscuous::Subscriber::Polymorphic
5
- include Promiscuous::Subscriber::AMQP
6
- include Promiscuous::Subscriber::Envelope
7
- include Promiscuous::Subscriber::Model
8
-
9
- def fetch
10
- klass.new.tap { |o| o.id = id if o.respond_to?(:id=) }
11
- end
12
-
13
- def commit
14
- with_dependencies do
15
- instance.run_callbacks operation unless operation == :dummy
16
- end
17
- end
18
-
19
- def self.subscribe(options)
20
- super
21
- raise "#{klass} must inherit from Promiscuous::Observer" unless klass < Promiscuous::Observer
22
-
23
- use_payload_attribute :id
24
- use_payload_attribute :operation, :symbolize => true
25
- end
26
- end
@@ -1,36 +0,0 @@
1
- module Promiscuous::Subscriber::Polymorphic
2
- extend ActiveSupport::Concern
3
- include Promiscuous::Subscriber::Envelope
4
- include Promiscuous::Common::ClassHelpers
5
-
6
- included do
7
- class_attribute :polymorphic_map
8
- use_option :from_type
9
- end
10
-
11
- module ClassMethods
12
- def from_type
13
- super ? super : guess_class_name('Subscribers')
14
- end
15
-
16
- def from_type=(value)
17
- super
18
- setup_polymorphic_mapping
19
- end
20
-
21
- def setup_polymorphic_mapping
22
- self.polymorphic_map ||= {}
23
- polymorphic_map[from_type.to_s] = self if from_type
24
- end
25
-
26
- def inherited(subclass)
27
- super
28
- subclass.setup_polymorphic_mapping unless options.has_key?(:class)
29
- end
30
-
31
- def polymorphic_subscriber_from(payload)
32
- type = payload.is_a?(Hash) ? payload['type'] : nil
33
- polymorphic_map[type] || self
34
- end
35
- end
36
- end
@@ -1,12 +0,0 @@
1
- module Promiscuous::Subscriber::Upsert
2
- extend ActiveSupport::Concern
3
-
4
- def fetch
5
- begin
6
- super
7
- rescue self.class.missing_record_exception
8
- Promiscuous.warn "[receive] upserting #{payload}"
9
- fetch_new
10
- end
11
- end
12
- end