aws_sqs_moniter 0.0.1 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/lib/aws_sqs_moniter.rb +26 -2
  3. data/lib/aws_sqs_moniter/aws/arns.rb +29 -0
  4. data/lib/aws_sqs_moniter/aws/builder.rb +48 -0
  5. data/lib/aws_sqs_moniter/aws/builder/application_policy_builder.rb +73 -0
  6. data/lib/aws_sqs_moniter/aws/builder/queue_builder.rb +100 -0
  7. data/lib/aws_sqs_moniter/aws/builder/subscription_builder.rb +48 -0
  8. data/lib/aws_sqs_moniter/aws/builder/topic_builder.rb +27 -0
  9. data/lib/aws_sqs_moniter/aws/environmental_name.rb +13 -0
  10. data/lib/aws_sqs_moniter/configuration.rb +110 -0
  11. data/lib/aws_sqs_moniter/configuration/queue_configuration.rb +49 -0
  12. data/lib/aws_sqs_moniter/configuration/redrive_policy_configuration.rb +33 -0
  13. data/lib/aws_sqs_moniter/configuration/validatable.rb +15 -0
  14. data/lib/aws_sqs_moniter/dead_letters/retrier.rb +23 -0
  15. data/lib/aws_sqs_moniter/dead_letters/worker.rb +34 -0
  16. data/lib/aws_sqs_moniter/logging.rb +66 -0
  17. data/lib/aws_sqs_moniter/middleware/server/active_record/connection_pool.rb +15 -0
  18. data/lib/aws_sqs_moniter/middleware/server/active_record/idempotence.rb +24 -0
  19. data/lib/aws_sqs_moniter/middleware/server/active_record/retrier.rb +23 -0
  20. data/lib/aws_sqs_moniter/middleware/server/active_record/transaction.rb +36 -0
  21. data/lib/aws_sqs_moniter/middleware/server/airbrake.rb +24 -0
  22. data/lib/aws_sqs_moniter/monkey_patches/forbid_implicit_active_record_connection_checkout.rb +34 -0
  23. data/lib/aws_sqs_moniter/railtie.rb +10 -0
  24. data/lib/aws_sqs_moniter/typed_message.rb +37 -0
  25. data/lib/aws_sqs_moniter/version.rb +1 -1
  26. data/lib/aws_sqs_moniter/worker_registries/typed_message_registry.rb +99 -0
  27. data/lib/generators/aws_sqs_moniter/install_generator.rb +30 -0
  28. data/lib/generators/aws_sqs_moniter/templates/create_dead_letters_migration.rb +11 -0
  29. data/lib/generators/aws_sqs_moniter/templates/create_processed_messages_migration.rb +13 -0
  30. data/lib/generators/aws_sqs_moniter/templates/create_published_messages_migration.rb +20 -0
  31. data/lib/generators/aws_sqs_moniter/templates/dead_letter.rb +14 -0
  32. data/lib/generators/aws_sqs_moniter/templates/initializer.rb +32 -0
  33. data/lib/generators/aws_sqs_moniter/templates/processed_message.rb +12 -0
  34. data/lib/generators/aws_sqs_moniter/templates/published_message.rb +14 -0
  35. data/lib/generators/aws_sqs_moniter/templates/shoryuken.yml +16 -0
  36. data/lib/tasks/aws_sqs_moniter.rake +83 -0
  37. data/lib/tasks/aws_sqs_setup.rake +10 -0
  38. metadata +52 -12
  39. data/.gitignore +0 -14
  40. data/Gemfile +0 -4
  41. data/LICENSE.txt +0 -22
  42. data/README.md +0 -31
  43. data/aws_sqs_moniter-0.0.1.gem +0 -0
  44. data/aws_sqs_moniter.gemspec +0 -36
  45. data/test/test_helper.rb +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 35611d2ac97dd5a262318bc8e10fa7d77766ba27
4
- data.tar.gz: 68b6834cef8d624d31e8f820234bc292dae9da75
3
+ metadata.gz: a99854dd21c0a9bce3dfd51f2608978d51154680
4
+ data.tar.gz: 1a760d80e4ce9e706ae510319898d226676102d4
5
5
  SHA512:
6
- metadata.gz: cd483589e313e71114ed14c9e560b7c460cd852d5bbb3207bc1e11523924c8e9ca69f7de16e860fff0b682330ee3b94c8db81c6221e423bcb1455cb036f7438e
7
- data.tar.gz: 14e42ca9c58c54b15c3874a2e056a98609705c84c570560d46acef7130547cc277d0d6d8c259c660a195b080d94bda880a0d9035c574458e57f7effdccd344ac
6
+ metadata.gz: f8b0dbc36da0ce368f5701381756ad76c93187c70b0f1875a89e9a4fb1ccb4338ab156e74eab11a306c17e011339c6cb3d997774905e272c6b35e9f8703d5905
7
+ data.tar.gz: d5dbc5cd2f0baec5d88dd3423da62c0d0650c384760478dfe30f06add657772b75b09eac027c1242f402d288b0675c2f2b1ab8176b5745a15126585d3ccf584c
@@ -1,5 +1,29 @@
1
- require "aws_sqs_moniter/version"
1
+ require 'shoryuken'
2
+ require 'aws_sqs_moniter/version'
3
+ require 'aws_sqs_moniter/logging'
4
+ require 'aws_sqs_moniter/configuration'
5
+ require 'aws_sqs_moniter/railtie' if defined? Rails
2
6
 
3
7
  module AwsSqsMoniter
4
- # Your code goes here...
8
+ class << self
9
+ def configure
10
+ @config ||= Configuration.new.tap do |config|
11
+ yield config
12
+ config.validate!
13
+ end
14
+ end
15
+
16
+ def configuration
17
+ fail 'You must call AwsSqsMoniter .configure in an initializer.' unless @config
18
+ @config
19
+ end
20
+
21
+ def sns_client
22
+ ::Aws::SNS::Client.new configuration.sns_options
23
+ end
24
+
25
+ def sqs_client
26
+ ::Aws::SQS::Client.new configuration.sqs_options
27
+ end
28
+ end
5
29
  end
@@ -0,0 +1,29 @@
1
+ module AwsSqsMoniter
2
+ module Aws
3
+ class Arns
4
+ def sns_arn
5
+ if AwsSqsMoniter.configuration.sns_protocol == 'cqs'
6
+ "arn:cmb:cns:ccp:#{AwsSqsMoniter.configuration.aws_account_id}"
7
+ else
8
+ "arn:aws:sns:#{AwsSqsMoniter.configuration.aws_region}:#{AwsSqsMoniter.configuration.aws_account_id}"
9
+ end
10
+ end
11
+
12
+ def sns_topic_arn topic
13
+ "#{sns_arn}:#{topic}"
14
+ end
15
+
16
+ def sqs_arn
17
+ if AwsSqsMoniter.configuration.sqs_protocol == 'cqs'
18
+ "arn:cmb:cqs:ccp:#{AwsSqsMoniter.configuration.aws_account_id}"
19
+ else
20
+ "arn:aws:sqs:#{AwsSqsMoniter.configuration.aws_region}:#{AwsSqsMoniter.configuration.aws_account_id}"
21
+ end
22
+ end
23
+
24
+ def sqs_queue_arn queue
25
+ "#{sqs_arn}:#{queue}"
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,48 @@
1
+ require_relative 'arns'
2
+ require_relative 'environmental_name'
3
+ require_relative 'builder/application_policy_builder'
4
+ require_relative 'builder/queue_builder'
5
+ require_relative 'builder/subscription_builder'
6
+ require_relative 'builder/topic_builder'
7
+
8
+ module AwsSqsMoniter
9
+ module Aws
10
+ class Builder
11
+ def initialize logger = nil
12
+ @logger = logger || Shoryuken::Logging.logger
13
+ end
14
+
15
+ def build_policies
16
+ ApplicationPolicyBuilder.new(@logger).build
17
+ end
18
+
19
+ def build_queues
20
+ Shoryuken.worker_registry.queues.each do |queue|
21
+ QueueBuilder.new(@logger, queue).build
22
+ end
23
+ end
24
+
25
+ def build_topics
26
+ Shoryuken.worker_registry.topics.each do |topic|
27
+ TopicBuilder.new(@logger, topic).build
28
+ end
29
+ end
30
+
31
+ def build_subscriptions
32
+ Shoryuken.worker_registry.queues.each do |queue|
33
+ SubscriptionBuilder.new(@logger, queue).build
34
+ end
35
+ end
36
+
37
+ def delete_all
38
+ Shoryuken.worker_registry.queues.each do |queue|
39
+ QueueBuilder.new(@logger, queue).delete
40
+ end
41
+
42
+ Shoryuken.worker_registry.topics.each do |topics|
43
+ TopicBuilder.new(@logger, topics).delete
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,73 @@
1
+ module AwsSqsMoniter
2
+ module Aws
3
+ class Builder
4
+ class ApplicationPolicyBuilder
5
+ def initialize logger
6
+ @logger = logger
7
+ @arns = Arns.new
8
+ end
9
+
10
+ def build
11
+ file = Tempfile.create ['aws-application-policy', '.json']
12
+ file.write application_policy
13
+ @logger.info application_policy_written_to: file.path
14
+ end
15
+
16
+ private
17
+
18
+ def application_policy
19
+ topic_arns = Shoryuken.worker_registry.topics.map { |topic| @arns.sns_topic_arn topic }
20
+ queue_arns = Shoryuken.worker_registry.queues.map { |queue| @arns.sqs_queue_arn queue }
21
+
22
+ <<-EOS
23
+ {
24
+ "Version": "2012-10-17",
25
+ "Statement": [
26
+ {
27
+ "Effect": "Allow",
28
+ "Action": [
29
+ "sns:CreateTopic",
30
+ "sns:Publish",
31
+ "sns:SetEndpointAttributes",
32
+ "sns:Subscribe"
33
+ ],
34
+ "Resource": [
35
+ #{arn_array_policy_string topic_arns, ' '}
36
+ ]
37
+ },
38
+ {
39
+ "Effect": "Allow",
40
+ "Action": [
41
+ "sns:SetSubscriptionAttributes"
42
+ ],
43
+ "Resource": [
44
+ "#{@arns.sns_topic_arn '*'}"
45
+ ]
46
+ },
47
+ {
48
+ "Effect": "Allow",
49
+ "Action": [
50
+ "sqs:ChangeMessageVisibility",
51
+ "sqs:CreateQueue",
52
+ "sqs:DeleteMessage",
53
+ "sqs:GetQueueAttributes",
54
+ "sqs:GetQueueUrl",
55
+ "sqs:ReceiveMessage",
56
+ "sqs:SetQueueAttributes"
57
+ ],
58
+ "Resource": [
59
+ #{arn_array_policy_string queue_arns, ' '}
60
+ ]
61
+ }
62
+ ]
63
+ }
64
+ EOS
65
+ end
66
+
67
+ def arn_array_policy_string arns, indent
68
+ arns.sort.map { |arn| %(#{indent}"#{arn}") }.join(",\n")
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,100 @@
1
+ module AwsSqsMoniter
2
+ module Aws
3
+ class Builder
4
+ class QueueBuilder
5
+ def initialize logger, name
6
+ @logger = logger
7
+ @name = name
8
+ @arns = Arns.new
9
+ @sqs = AwsSqsMoniter.sqs_client
10
+ end
11
+
12
+ def build
13
+ @logger.info building_queue: @name
14
+
15
+ queue_url = @sqs.create_queue(queue_name: @name)[:queue_url]
16
+
17
+ queue_arn = @sqs.get_queue_attributes(
18
+ queue_url: queue_url,
19
+ attribute_names: ['QueueArn']).attributes['QueueArn']
20
+
21
+ queue_configuration = AwsSqsMoniter.configuration.get_queue(@name)
22
+
23
+ attributes = {
24
+ 'DelaySeconds' => queue_configuration.delay_seconds.to_s,
25
+ 'MessageRetentionPeriod' => queue_configuration.message_retention_period.to_s,
26
+ 'VisibilityTimeout' => queue_configuration.visibility_timeout.to_s
27
+ }
28
+
29
+ if AwsSqsMoniter.configuration.sqs_protocol == 'sqs'
30
+ if queue_configuration.respond_to?(:redrive_policy)
31
+ policy = '{}'
32
+ policy = redrive_policy(
33
+ queue_configuration.redrive_policy.dead_letter_queue,
34
+ queue_configuration.redrive_policy.max_receive_count) if queue_configuration.redrive_policy.enabled
35
+
36
+ attributes.merge! 'RedrivePolicy' => policy
37
+ end
38
+
39
+ policy = queue_policy
40
+ attributes.merge! 'Policy' => policy
41
+ end
42
+
43
+ @sqs.set_queue_attributes queue_url: queue_url, attributes: attributes
44
+ end
45
+
46
+ def delete
47
+ @logger.info deleting_queue: @name
48
+
49
+ queue_url = @sqs.get_queue_url(queue_name: @name)[:queue_url]
50
+ @sqs.delete_queue(queue_url: queue_url)
51
+ rescue ::Aws::SQS::Errors::NonExistentQueue
52
+ end
53
+
54
+ private
55
+
56
+ def arn_array_policy_string arns, indent
57
+ arns.sort.map { |arn| %(#{indent}"#{arn}") }.join(",\n")
58
+ end
59
+
60
+ def queue_policy
61
+ queue_arn = @arns.sqs_queue_arn @name
62
+ topic_arns = topics.map { |topic| @arns.sns_topic_arn topic }
63
+
64
+ <<-EOS
65
+ {
66
+ "Version": "2008-10-17",
67
+ "Id": "#{queue_arn}/envoy-generated-policy",
68
+ "Statement": [
69
+ {
70
+ "Effect": "Allow",
71
+ "Principal": {
72
+ "AWS": "*"
73
+ },
74
+ "Action": "SQS:SendMessage",
75
+ "Resource": "#{queue_arn}",
76
+ "Condition": {
77
+ "ArnEquals": {
78
+ "aws:SourceArn": [
79
+ #{arn_array_policy_string topic_arns, ' '}
80
+ ]
81
+ }
82
+ }
83
+ }
84
+ ]
85
+ }
86
+ EOS
87
+ end
88
+
89
+ def redrive_policy dead_letter_queue, max_receive_count
90
+ arn = @arns.sqs_queue_arn EnvironmentalName.new(dead_letter_queue).to_s
91
+ %({"maxReceiveCount":"#{max_receive_count}", "deadLetterTargetArn":"#{arn}"})
92
+ end
93
+
94
+ def topics
95
+ Shoryuken.worker_registry.topics(@name)
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,48 @@
1
+ module AwsSqsMoniter
2
+ module Aws
3
+ class Builder
4
+ class SubscriptionBuilder
5
+ def initialize logger, queue
6
+ @logger = logger
7
+ @queue = queue
8
+ @arns = Arns.new
9
+ @sns = AwsSqsMoniter.sns_client
10
+ @sqs = AwsSqsMoniter.sqs_client
11
+ end
12
+
13
+ def build
14
+ queue_url = @sqs.get_queue_url(queue_name: @queue)[:queue_url]
15
+
16
+ queue_arn = @sqs.get_queue_attributes(
17
+ queue_url: queue_url,
18
+ attribute_names: ['QueueArn']).attributes['QueueArn']
19
+
20
+ topics.each do |topic|
21
+ topic = EnvironmentalName.new(topic).to_s
22
+ topic_arn = @arns.sns_topic_arn topic
23
+
24
+ if AwsSqsMoniter.configuration.sns_protocol == 'cqs'
25
+ @sns.add_permission topic_arn: topic_arn,
26
+ label: "subscribe-#{AwsSqsMoniter.configuration.aws_account_id}-#{Time.now.strftime('%Y%m%d%H%M%S')}",
27
+ aws_account_id: [AwsSqsMoniter.configuration.aws_account_id],
28
+ action_name: ['Subscribe']
29
+ end
30
+
31
+ @logger.info subscribing_queue: @queue, subscription_topic: topic
32
+
33
+ @sns.subscribe(
34
+ endpoint: queue_arn,
35
+ protocol: AwsSqsMoniter.configuration.sns_protocol,
36
+ topic_arn: topic_arn)
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def topics
43
+ Shoryuken.worker_registry.topics(@queue)
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,27 @@
1
+ module AwsSqsMoniter
2
+ module Aws
3
+ class Builder
4
+ class TopicBuilder
5
+ def initialize logger, name
6
+ @logger = logger
7
+ @name = EnvironmentalName.new(name).to_s
8
+ @sns = AwsSqsMoniter.sns_client
9
+ @arns = Arns.new
10
+ end
11
+
12
+ def build
13
+ @logger.info building_topic: @name
14
+
15
+ @sns.create_topic name: @name
16
+ end
17
+
18
+ def delete
19
+ @logger.info deleting_topic: @name
20
+
21
+ @sns.delete_topic topic_arn: @arns.sns_topic_arn(@name)
22
+ rescue ::Aws::SNS::Errors::NotFound
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,13 @@
1
+ module AwsSqsMoniter
2
+ module Aws
3
+ class EnvironmentalName
4
+ def initialize(name)
5
+ @name = [name.to_s.dasherize, Rails.env].join('-')
6
+ end
7
+
8
+ def to_s
9
+ @name
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,110 @@
1
+ require_relative 'configuration/validatable'
2
+ require_relative 'configuration/queue_configuration'
3
+ require_relative 'configuration/redrive_policy_configuration'
4
+
5
+ module AwsSqsMoniter
6
+ class Configuration
7
+ include Validatable
8
+
9
+ def initialize
10
+ @queues = {}
11
+ @queue_defaults = QueueConfiguration.new 'queue_defaults'
12
+ @sns_protocol = @sqs_protocol = 'sqs'
13
+ end
14
+
15
+ attr_reader :queue_defaults, :sns_endpoint, :sns_protocol, :sqs_endpoint, :sqs_protocol
16
+
17
+ def add_queue name
18
+ environmental_name = Aws::EnvironmentalName.new(name).to_s
19
+ fail "A queue called #{name} already exists." if @queues.key? environmental_name
20
+
21
+ queue = QueueConfiguration.new name
22
+ @queue_defaults.copy_onto queue
23
+ yield queue if block_given?
24
+ @queues[environmental_name] = queue
25
+ end
26
+
27
+ def aws_options
28
+ @aws_options ||= shoryuken_config[:aws].slice :access_key_id, :region, :secret_access_key
29
+ end
30
+
31
+ def aws_account_id
32
+ shoryuken_config[:aws][:account_id]
33
+ end
34
+
35
+ def aws_region
36
+ aws_options[:region]
37
+ end
38
+
39
+ def enable_cmb_mode
40
+ @sns_protocol = @sqs_protocol = 'cqs'
41
+ @sns_endpoint = shoryuken_config[:aws][:sns_endpoint]
42
+ @sqs_endpoint = shoryuken_config[:aws][:sqs_endpoint]
43
+ end
44
+
45
+ def queues
46
+ @queues.values
47
+ end
48
+
49
+ def get_queue environmental_name
50
+ @queues[environmental_name]
51
+ end
52
+
53
+ def require_workers
54
+ Dir[Rails.root + 'app' + 'workers' + '*.rb'].each do |path|
55
+ require path
56
+ end
57
+ end
58
+
59
+ def shoryuken_config
60
+ @shoryuken_config ||= begin
61
+ YAML.load(
62
+ ERB.new(
63
+ IO.read(
64
+ Rails.root + 'config' + 'shoryuken.yml')
65
+ ).result
66
+ ).with_indifferent_access
67
+ end
68
+ end
69
+
70
+ def sns_options
71
+ aws_client_options :sns_endpoint
72
+ end
73
+
74
+ def sqs_options
75
+ aws_client_options :sqs_endpoint
76
+ end
77
+
78
+ def use_aws_sqs_moniter_logging
79
+ require 'aws_sqs_moniter/logging'
80
+ Shoryuken::Logging.logger = AwsSqsMoniter::Logging.logger
81
+ end
82
+
83
+ def use_typed_message_registry dead_letter_queue_name
84
+ require 'aws_sqs_moniter/worker_registries/typed_message_registry'
85
+ Shoryuken.worker_registry = AwsSqsMoniter::WorkerRegistries::TypedMessageRegistry.new dead_letter_queue_name
86
+ end
87
+
88
+ def validate
89
+ [queue_defaults, queues].flatten.compact.each do |section|
90
+ section.valid?
91
+ errors.push *(section.errors)
92
+ end
93
+ end
94
+
95
+ def validate!
96
+ return if valid?
97
+
98
+ fail %(Invalid configuration: \n\n#{errors.join("\n")}\n\n)
99
+ end
100
+
101
+ private
102
+
103
+ def aws_client_options endpoint_key
104
+ options = aws_options
105
+ endpoint = shoryuken_config[:aws][endpoint_key]
106
+ options = options.merge(endpoint: endpoint) unless endpoint.blank?
107
+ options
108
+ end
109
+ end
110
+ end