aws_sqs_moniter 0.0.1 → 0.0.3

data/lib/aws_sqs_moniter/version.rb

@@ -1,3 +1,3 @@
 module AwsSqsMoniter
-  VERSION = "0.0.1"
+  VERSION = '0.0.3'
 end

data/lib/aws_sqs_moniter/worker_registries/typed_message_registry.rb

@@ -0,0 +1,99 @@
+require 'aws_sqs_moniter/aws/environmental_name'
+require 'aws_sqs_moniter/typed_message'
+
+module AwsSqsMoniter
+  module WorkerRegistries
+    class TypedMessageRegistry < Shoryuken::WorkerRegistry
+      DuplicateSubscriptionError = Class.new StandardError
+      InvalidWorkerOptionsError = Class.new StandardError
+      UnroutableMessageError = Class.new StandardError
+      WorkerNotFoundError = Class.new StandardError
+
+      def initialize(dead_letter_queue_name)
+        @subscriptions = {}
+        @dead_letter_queue_name = dead_letter_queue_name
+      end
+
+      def batch_receive_messages?(_queue)
+        false
+      end
+
+      def clear
+        @subscriptions.clear
+      end
+
+      def fetch_worker(queue, message)
+        queue_subscriptions = @subscriptions.fetch(queue) do
+          fail UnroutableMessageError, "#{self} does not know how to route messages for queue '#{queue}'."
+        end
+
+        message = TypedMessage.new message
+        worker_class = queue_subscriptions[message.type] || queue_subscriptions['*']
+
+        fail WorkerNotFoundError,
+             "Worker not found for message type #{message.type} on queue #{queue}." unless worker_class
+
+        worker_class.new
+      end
+
+      def queues
+        @subscriptions.keys
+      end
+
+      def register_worker(queue, clazz)
+        invalid_worker = workers(queue).find do |worker_class|
+          worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
+        end
+
+        fail ArgumentError, "Could not register #{clazz} for '#{queue}', "\
+          "because #{invalid_worker} is already registered for this queue, "\
+          "and Shoryuken doesn't support a batchable worker for a queue with multiple workers" if invalid_worker
+
+        worker_subscriptions = clazz.get_shoryuken_options['subscriptions']
+
+        if worker_subscriptions.nil?
+          fail InvalidWorkerOptionsError, "Worker #{clazz} must define "\
+            'a :subscriptions hash ({ queue_name: message_types }) in '\
+            "it's shoryuken_options"
+        end
+
+        worker_subscriptions.each do |owning_queue, message_types|
+          queue_subscriptions = @subscriptions.fetch_store env_name(owning_queue), {}
+
+          [message_types].flatten.each do |message_type|
+            message_type = message_type.to_s.dasherize
+
+            if queue_subscriptions.key? message_type
+              fail DuplicateSubscriptionError, "Worker #{clazz} cannot "\
+                "define another subscription of message #{message_type} "\
+                "on queue #{owning_queue} as it is already subscribed to by "\
+                "worker #{queue_subscriptions[message_type]}."
+            end
+
+            queue_subscriptions.store message_type, clazz
+          end
+        end
+      end
+
+      def topics queue = nil
+        topic_list = if queue
+                       @subscriptions.fetch(queue, {}).keys
+                     else
+                       @subscriptions.values.map(&:keys).flatten
+                     end
+
+        topic_list.uniq - ['*']
+      end
+
+      def workers(queue)
+        @subscriptions.fetch(queue, {}).values
+      end
+
+      private
+
+      def env_name(name)
+        AwsSqsMoniter::Aws::EnvironmentalName.new(name.to_s).to_s
+      end
+    end
+  end
+end

data/lib/generators/aws_sqs_moniter/install_generator.rb

@@ -0,0 +1,30 @@
+require 'rails/generators/named_base'
+require 'rails/generators/active_record/migration'
+
+module AwsSqsMoniter
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    desc 'Installs models and initializers to Rails Application'
+
+    source_root File.expand_path('../templates', __FILE__)
+
+    def install_files
+      #copiong initializer to config/intializer of rails application
+      copy_file 'initializer.rb', 'config/initializers/aws_sqs_moniter_intializer.rb'
+      copy_file 'dead_letter.rb', 'app/models/dead_letter.rb'
+      copy_file 'processed_message.rb', 'app/models/processed_message.rb'
+      copy_file 'published_message.rb', 'app/models/published_message.rb'
+      #coping aws config file to  rails application
+      copy_file 'shoryuken.yml' , 'config/shoryuken.yml'
+      migration_template 'create_dead_letters_migration.rb',
+                         'db/migrate/create_dead_letters.rb'
+
+      migration_template 'create_processed_messages_migration.rb',
+                         'db/migrate/create_processed_messages.rb'
+
+      migration_template 'create_published_messages_migration.rb',
+                         'db/migrate/create_published_messages.rb'
+    end
+  end
+end

data/lib/generators/aws_sqs_moniter/templates/create_dead_letters_migration.rb

@@ -0,0 +1,11 @@
+class CreateDeadLetters < ActiveRecord::Migration
+  def change
+    create_table :dead_letters, id: false do |t|
+      t.timestamps
+
+      t.string :sqs_id, null: false, index:  { unique: true }
+      t.uuid :message_id, null: false, index: true
+      t.json :message, null: false
+    end
+  end
+end

data/lib/generators/aws_sqs_moniter/templates/create_processed_messages_migration.rb

@@ -0,0 +1,13 @@
+class CreateProcessedMessages < ActiveRecord::Migration
+  def change
+    create_table :processed_messages, id: :uuid do |t|
+      t.timestamps
+
+      t.uuid :message_id, null: false
+      t.string :queue, null: false
+      t.json :message, null: false
+
+      t.index %i(message_id queue), unique: true
+    end
+  end
+end

data/lib/generators/aws_sqs_moniter/templates/create_published_messages_migration.rb

@@ -0,0 +1,20 @@
+class CreatePublishedMessages < ActiveRecord::Migration
+  def change
+    create_table :published_messages, id: :uuid do |t|
+      t.timestamps
+
+      t.string :topic, null: false
+      t.json :message, null: false
+      t.json :response
+
+      t.integer :attempts, null: false, default: 0
+      t.datetime :attempted_at
+
+      t.string :published_by
+      t.datetime :published_at
+
+      t.index :published_by
+      t.index :published_at
+    end
+  end
+end

data/lib/generators/aws_sqs_moniter/templates/dead_letter.rb

@@ -0,0 +1,14 @@
+class DeadLetter < ActiveRecord::Base
+  self.primary_key = 'sqs_id'
+
+  validates :sqs_id, presence: true
+  validates :message_id, presence: true
+  validates :message, presence: true
+
+  def self.retryable sqs_id: nil, message_id: nil
+    criteria = all
+    criteria = criteria.where(sqs_id: sqs_id) unless sqs_id.blank?
+    criteria = criteria.where(message_id: message_id) unless message_id.blank?
+    criteria
+  end
+end

data/lib/generators/aws_sqs_moniter/templates/initializer.rb

@@ -0,0 +1,32 @@
+# Load any middlewares you want to use:
+#
+# require 'aws_sqs_moniter/middleware/server/airbrake'
+# require 'aws_sqs_moniter/middleware/server/active_record/connection_pool'
+
+# Configure your environment:
+#
+# AwsSqsMoniter.configure do |config|
+#   config.use_aws_sqs_moniter_logging
+#   config.use_typed_message_registry 'your_dead_letter_queue_name'
+
+#   if Rails.env.development?
+#     config.enable_cmb_mode
+#     config.require_application_workers
+#     config.require_dead_letter_worker
+#   end
+
+#   config.queue_defaults.delay_seconds = 0
+#   config.queue_defaults.message_retention_period = 1_209_600
+#   config.queue_defaults.visibility_timeout = 30
+
+#   config.add_queue('your_dead_letter_queue_name') do |queue|
+#     queue.visibility_timeout = 60
+#   end
+
+#   config.add_queue('your_queue_name') do |queue|
+#     queue.delay_seconds = 10
+#     queue.redrive_policy.dead_letter_queue = 'your_dead_letter_queue_name'
+#     queue.redrive_policy.enabled = true
+#     queue.redrive_policy.max_receive_count = 5
+#   end
+# end

data/lib/generators/aws_sqs_moniter/templates/processed_message.rb

@@ -0,0 +1,12 @@
+class ProcessedMessage < ActiveRecord::Base
+  validates :message_id, presence: true
+  validates :queue, presence: true
+  validates :message, presence: true
+
+  def self.log message
+    create!(
+      message_id: message.id,
+      queue: message.queue,
+      message: message.to_h)
+  end
+end

data/lib/generators/aws_sqs_moniter/templates/published_message.rb

@@ -0,0 +1,14 @@
+class PublishedMessage < ActiveRecord::Base
+  def self.unsent
+    where(published_at: nil)
+      .where('attempted_at IS NULL OR (attempted_at <= ?)', 1.minute.ago)
+      .order(created_at: :asc)
+  end
+
+  def self.sent_by publisher_id, timestamp
+    where(published_by: publisher_id, attempted_at: timestamp)
+  end
+
+  validates :message, presence: true
+  validates :topic, presence: true
+end

data/lib/generators/aws_sqs_moniter/templates/shoryuken.yml

@@ -0,0 +1,16 @@
+#AWS credentails need to keep here
+aws:
+  access_key_id:      ""       # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
+  secret_access_key:  ""       # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
+  region:             us-east-1 # or <%= ENV['AWS_REGION'] %>
+  receive_message:              # See http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/Queue.html#receive_message-instance_method
+    # wait_time_seconds: N      # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
+    attributes:
+      - receive_count
+      - sent_at
+concurrency: 25,  # The number of allocated threads to process messages. Default 25
+delay: 25,        # The delay in seconds to pause a queue when it's empty. Default 0
+queues:
+  - [high_priority, 6]
+  - [default, 2]
+  - [low_priority, 1]

data/lib/tasks/aws_sqs_moniter.rake

@@ -0,0 +1,83 @@
+namespace :aws_sqs_moniter do
+  namespace :aws do
+    namespace :build do
+      desc 'Builds policies, topics and queues'
+      task all: %i(environment topics queues subscriptions policies) do
+      end
+
+      desc 'Builds the application policies'
+      task policies: :environment do
+        builder.build_policies
+      end
+
+      desc 'Builds the queues'
+      task queues: :environment do
+        builder.build_queues
+      end
+
+      desc 'Builds the subscriptions'
+      task subscriptions: :environment do
+        builder.build_subscriptions
+      end
+
+      desc 'Builds the topics'
+      task topics: :environment do
+        builder.build_topics
+      end
+
+      private
+
+      def builder
+        AwsSqsMoniter.configuration.require_workers
+        require 'aws_sqs_moniter/aws/builder'
+        AwsSqsMoniter::Aws::Builder.new
+      end
+    end
+
+    namespace :destroy do
+      desc 'Deletes all topics and queues'
+      task all: :environment do
+        fail 'This task is only available in development.' unless Rails.env.development?
+        builder.delete_all
+      end
+    end
+  end
+
+  #Queues GET/READ/DELETE
+  namespace :queues do 
+
+    desc 'get queues'     
+    task get: :environment do
+    puts "Get Queue list from AWS SQS Account"
+    end
+
+    desc "delete queues"
+    task delete: :environment do
+    puts "Delete Queues from AWS SQS Account"
+    end
+
+    desc "READ messages from a queue"
+    task read: :environment do
+    puts "read messages from a queue"
+    end 
+
+  end
+
+  namespace :workers do
+    
+    desc "Start Workers"
+    task start: :environment do
+      puts "start workers in Background"
+    end
+  
+  end
+
+
+  namespace :dead_letters do
+    desc 'Retries all dead letters (or pass SQS_ID or MESSAGE_ID to select a specific message).'
+    task retry: :environment do
+      scope = DeadLetter.retriable sqs_id: ENV['SQS_ID'], message_id: ENV['MESSAGE_ID']
+      AwsSqsMoniter::DeadLetterRetrier.new.retry scope
+    end
+  end
+end

data/lib/tasks/aws_sqs_setup.rake

@@ -0,0 +1,10 @@
+namespace :aws_sqs_moniter do
+  desc 'setup Aws SQS on your machicne.'
+  task setup: :environment do
+    puts "coping SQS config file for keeping AWS credentails"
+  end
+
+  task run_migrations: :environment do 
+    puts "adding migrations to store/persist queue state/status."
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws_sqs_moniter
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - saikiranmothe
@@ -206,6 +206,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: docsplit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: AWS SQS Moniter.
 email:
 - saikiran.mothe@gmail.com
@@ -213,16 +227,43 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- Gemfile
-- LICENSE.txt
-- README.md
-- Rakefile
-- aws_sqs_moniter-0.0.1.gem
-- aws_sqs_moniter.gemspec
-- lib/aws_sqs_moniter.rb
+- lib/aws_sqs_moniter/monkey_patches/forbid_implicit_active_record_connection_checkout.rb
+- lib/aws_sqs_moniter/dead_letters/retrier.rb
+- lib/aws_sqs_moniter/dead_letters/worker.rb
 - lib/aws_sqs_moniter/version.rb
-- test/test_helper.rb
+- lib/aws_sqs_moniter/middleware/server/airbrake.rb
+- lib/aws_sqs_moniter/middleware/server/active_record/transaction.rb
+- lib/aws_sqs_moniter/middleware/server/active_record/retrier.rb
+- lib/aws_sqs_moniter/middleware/server/active_record/idempotence.rb
+- lib/aws_sqs_moniter/middleware/server/active_record/connection_pool.rb
+- lib/aws_sqs_moniter/configuration.rb
+- lib/aws_sqs_moniter/configuration/redrive_policy_configuration.rb
+- lib/aws_sqs_moniter/configuration/validatable.rb
+- lib/aws_sqs_moniter/configuration/queue_configuration.rb
+- lib/aws_sqs_moniter/worker_registries/typed_message_registry.rb
+- lib/aws_sqs_moniter/logging.rb
+- lib/aws_sqs_moniter/typed_message.rb
+- lib/aws_sqs_moniter/railtie.rb
+- lib/aws_sqs_moniter/aws/arns.rb
+- lib/aws_sqs_moniter/aws/builder.rb
+- lib/aws_sqs_moniter/aws/builder/application_policy_builder.rb
+- lib/aws_sqs_moniter/aws/builder/topic_builder.rb
+- lib/aws_sqs_moniter/aws/builder/queue_builder.rb
+- lib/aws_sqs_moniter/aws/builder/subscription_builder.rb
+- lib/aws_sqs_moniter/aws/environmental_name.rb
+- lib/aws_sqs_moniter.rb
+- lib/tasks/aws_sqs_moniter.rake
+- lib/tasks/aws_sqs_setup.rake
+- lib/generators/aws_sqs_moniter/install_generator.rb
+- lib/generators/aws_sqs_moniter/templates/initializer.rb
+- lib/generators/aws_sqs_moniter/templates/shoryuken.yml
+- lib/generators/aws_sqs_moniter/templates/create_published_messages_migration.rb
+- lib/generators/aws_sqs_moniter/templates/create_dead_letters_migration.rb
+- lib/generators/aws_sqs_moniter/templates/processed_message.rb
+- lib/generators/aws_sqs_moniter/templates/dead_letter.rb
+- lib/generators/aws_sqs_moniter/templates/published_message.rb
+- lib/generators/aws_sqs_moniter/templates/create_processed_messages_migration.rb
+- Rakefile
 homepage: http://saikiranmothe.github.io/aws_sqs_moniter
 licenses:
 - MIT
@@ -247,6 +288,5 @@ rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: AWS SQS Moniter
-test_files:
-- test/test_helper.rb
+test_files: []
 has_rdoc: