eventq_aws 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72dc6e5eedbb1a486df5fb6ae630c9ada26a08b5
+  data.tar.gz: 953364700ea24b683038d3fbd4b2dd5dffbfa7bd
+SHA512:
+  metadata.gz: 66b70231f26a2723ff5a37104991d90d562ae6c97e52619472e174aff95e49b53a5f3362bb97718668fc8e2c070746dde9a9426a1086fb784176dfeb87518814
+  data.tar.gz: 0633ab90abfc5d3accfd9803ce68967f10ce5de746ef29fa40b49335e7bd73805c7790c32d1e0f9707872f930e7138a0c512ad7e6b4caf3e9ffbd5633b6b3f0c

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "eventq_aws"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/eventq_aws/aws_eventq_client.rb

@@ -0,0 +1,28 @@
+require 'oj'
+
+class AwsEventQClient
+  def initialize
+    @client = AwsQueueClient.new
+  end
+
+  def raise(event_type, event)
+
+    topic_arn = @client.get_topic_arn(event_type)
+
+    qm = QueueMessage.new
+    qm.content = event
+    qm.type = event_type
+
+    message = Oj.dump(qm)
+
+    response = @client.sns.publish({
+        topic_arn: topic_arn,
+        message: message,
+        subject: event_type
+                    })
+
+    return response.message_id
+
+  end
+
+end

data/lib/eventq_aws/aws_queue_client.rb

@@ -0,0 +1,28 @@
+class AwsQueueClient
+
+  attr_reader :sns
+  attr_reader :sqs
+
+  def initialize
+
+    Aws.config[:credentials] = Aws::Credentials.new(ENV['AWS_KEY'], ENV['AWS_SECRET'])
+    Aws.config[:region] = ENV['AWS_REGION'] || 'us-west-2'
+
+    @sns = Aws::SNS::Client.new
+    @sqs = Aws::SQS::Client.new
+
+    @aws_account = ENV['AWS_ACCOUNT_NUMBER']
+    @aws_region = ENV['AWS_REGION'] || 'us-west-2'
+
+  end
+
+  def get_topic_arn(event_type)
+    response = @sns.create_topic({ name: event_type })
+    return response.topic_arn
+  end
+
+  def get_queue_arn(queue)
+    return "arn:aws:sqs:#{@aws_region}:#{@aws_account}:#{queue.name}"
+  end
+
+end

data/lib/eventq_aws/aws_queue_manager.rb

@@ -0,0 +1,35 @@
+class AwsQueueManager
+
+  @@dead_letter_queue = 'dead_letter_archive'
+
+  def initialize
+    @client = AwsQueueClient.new
+  end
+
+  def get_queue(queue)
+
+    response = @client.sqs.create_queue({
+        queue_name: queue.name,
+        attributes: {
+            'VisibilityTimeout' => (queue.retry_delay / 1000).to_s
+        }
+                         })
+
+    return response.queue_url
+
+  end
+
+  def drop_queue(queue)
+
+    q = get_queue(queue)
+
+    @client.sqs.delete_queue({ queue_url: q})
+
+  end
+
+  def drop_topic(event_type)
+    topic_arn = @client.get_topic_arn(event_type)
+    @client.sns.delete_topic({ topic_arn: topic_arn})
+  end
+
+end

data/lib/eventq_aws/aws_queue_worker.rb

@@ -0,0 +1,145 @@
+class AwsQueueWorker
+
+  attr_accessor :is_running
+
+  def initialize
+    @threads = []
+    @is_running = false
+
+    @retry_exceeded_block = nil
+  end
+
+  def start(queue, options = {}, &block)
+    configure(queue, options)
+
+    puts '[QUEUE_WORKER] Listening for messages.'
+
+    raise 'Worker is already running.' if running?
+
+    @is_running = true
+    @threads = []
+
+    #loop through each thread count
+    @thread_count.times do
+      thr = Thread.new do
+
+        client = AwsQueueClient.new
+        manager = AwsQueueManager.new
+
+        #begin the queue loop for this thread
+        while true do
+
+          #check if the worker is still allowed to run and break out of thread loop if not
+          if !@is_running
+            break
+          end
+
+          #get the queue
+          q = manager.get_queue(queue)
+
+          received = false
+          error = false
+
+          begin
+
+            #request a message from the queue
+            response = client.sqs.receive_message({
+                queue_url: q,
+                max_number_of_messages: 1,
+                wait_time_seconds: 1,
+                message_attribute_names: ['ApproximateReceiveCount']
+                                              })
+
+            #check that a message was received
+            if response.messages.length > 0
+
+              msg = response.messages[0]
+              retry_attempts = msg.message_attributes['ApproximateReceiveCount'].to_i
+
+              #deserialize the message payload
+              message = Oj.load(msg.body)
+              payload = Oj.load(message["Message"])
+
+              puts "[QUEUE_WORKER] Message received. Retry Attempts: #{retry_attempts}"
+
+              #begin worker block for queue message
+              begin
+                block.call(payload.content, payload.type, retry_attempts)
+
+                #accept the message as processed
+                client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+                puts '[QUEUE_WORKER] Message acknowledged.'
+                received = true
+              rescue => e
+                puts '[QUEUE_WORKER] An unhandled error happened attempting to process a queue message.'
+                puts "Error: #{e}"
+
+                error = true
+                puts '[QUEUE_WORKER] Message rejected.'
+
+                if !queue.allow_retry
+                  #remove the message from the queue so that it does not get retried
+                  client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+                end
+
+              end
+
+            end
+
+          rescue
+            puts 'An error occured attempting to retrieve a message from the queue.'
+          end
+
+          #check if any message was received
+          if !received && !error
+            puts "[QUEUE_WORKER] No message received. Sleeping for #{@sleep} seconds"
+            #no message received so sleep before attempting to pop another message from the queue
+            sleep(@sleep)
+          end
+
+        end
+
+      end
+      @threads.push(thr)
+
+    end
+
+    if options.key?(:wait) && options[:wait] == true
+      @threads.each { |thr| thr.join }
+    end
+  end
+
+  def stop
+    @is_running = false
+    @threads.each { |thr| thr.join }
+  end
+
+  def on_retry_exceeded(&block)
+    @retry_exceeded_block = block
+  end
+
+  def running?
+    @is_running
+  end
+
+  private
+
+  def configure(queue, options = {})
+
+    @queue = queue
+
+    #default thread count
+    @thread_count = 5
+    if options.key?(:thread_count)
+      @thread_count = options[:thread_count]
+    end
+
+    #default sleep time in seconds
+    @sleep = 15
+    if options.key?(:sleep)
+      @sleep = options[:sleep]
+    end
+
+  end
+
+end

data/lib/eventq_aws/aws_subscription_manager.rb

@@ -0,0 +1,53 @@
+class AwsSubscriptionManager
+
+  def initialize
+    @client = AwsQueueClient.new
+    @manager = AwsQueueManager.new
+  end
+
+  def subscribe(event_type, queue)
+
+    topic_arn = @client.get_topic_arn(event_type)
+
+    q = @manager.get_queue(queue)
+    queue_arn = @client.get_queue_arn(queue)
+
+    @client.sqs.set_queue_attributes({
+        queue_url: q,
+        attributes:{
+            'Policy' => '{
+  "Version": "2012-10-17",
+  "Id": "SNStoSQS",
+  "Statement": [
+    {
+      "Sid":"rule1",
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:*",
+      "Resource": "' + queue_arn + '",
+      "Condition" : {
+        "ArnEquals" : {
+          "aws:SourceArn":"' + topic_arn + '"
+        }
+      }
+    }
+  ]
+}'
+        }
+                                     })
+
+    @client.sns.subscribe({
+        topic_arn: topic_arn,
+        protocol: 'sqs',
+        endpoint: queue_arn
+                      })
+
+  end
+
+  def unsubscribe(queue)
+
+    raise 'Not implemented. Please unsubscribe the queue from the topic inside the AWS Management Console.'
+
+  end
+
+end

data/lib/eventq_aws/version.rb

@@ -0,0 +1,3 @@
+module EventqAws
+  VERSION = "0.1.0"
+end

data/lib/eventq_aws.rb

@@ -0,0 +1,6 @@
+require "eventq_aws/version"
+require 'eventq_aws/aws_eventq_client'
+require 'eventq_aws/aws_queue_client'
+require 'eventq_aws/aws_queue_manager'
+require 'eventq_aws/aws_queue_worker'
+require 'eventq_aws/aws_subscription_manager'

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: eventq_aws
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- vaughanbrittonsage
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventq_base
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This is the aws implementation for EventQ
+email:
+- vaughanbritton@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/console
+- bin/setup
+- lib/eventq_aws.rb
+- lib/eventq_aws/aws_eventq_client.rb
+- lib/eventq_aws/aws_queue_client.rb
+- lib/eventq_aws/aws_queue_manager.rb
+- lib/eventq_aws/aws_queue_worker.rb
+- lib/eventq_aws/aws_subscription_manager.rb
+- lib/eventq_aws/version.rb
+homepage: https://github.com/vaughanbrittonsage/eventq
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: This is the aws implementation for EventQ
+test_files: []
+has_rdoc: