qprocessor 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e127381632515acbf030a9eca5338928a298187b155950102a33bff338af4fc
-  data.tar.gz: 767a5e859b1de2fcc61463b2b0c2d350a4ca7dbffc7ede0d352005fab68707cd
+  metadata.gz: acc989a7981c73a9503e369c54b4c912f3dab1ca073c33d49a38c379db2c0b7b
+  data.tar.gz: c408d6c67f41b2e0da1992aa8226d3b1bdd3e24c6ac9eda8bb91ed4010061af3
 SHA512:
-  metadata.gz: de8a0421e4fd44bb25badc482f8aa1a6b160f0d73594e3d3a689f9e538e57bcd37b0191d168206546776cc7a55a6e054e495db3a0bbeb0f90561c9087e4fe19f
-  data.tar.gz: fbbbd2241b98d85028964409991fcf87ddde61cfecaddc4ca99677717c828a2a1d6604e4bb3232f438aebc3ee742f0dbcb41520c3a7afe55e4ef18f38333d8fb
+  metadata.gz: 62068b04ecf60602b1094106c9fc0700659af07aa779b8f4b89e608a81f95e60fd29ef93257eee67f2cd0e1fa271bf25717017beed998b3b133c45a86f2e3d5d
+  data.tar.gz: ddeb4a8105387ec43a35a07edae349f6952a71e9b15d64b6e03b6bf0414bccfa57523067984610c36ef8422ff745358ac56ccd20e3c77c469e5006c30df6ac44

data/Gemfile.lock

@@ -1,13 +1,31 @@
 PATH
   remote: .
   specs:
-    qprocessor (0.1.0)
+    qprocessor (0.2.0)
+      aws-sdk-sqs (~> 1.49)
+      aws-sdk-sts (~> 1.5)
       beaneater (~> 1.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    aws-eventstream (1.2.0)
+    aws-partitions (1.548.0)
+    aws-sdk-core (3.125.3)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.525.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.49.0)
+      aws-sdk-core (~> 3, >= 3.125.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-sts (1.5.0)
+      aws-sdk-core (~> 3, >= 3.110.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.4.0)
+      aws-eventstream (~> 1, >= 1.0.2)
     beaneater (1.1.1)
+    jmespath (1.5.0)
     rake (13.0.6)
 
 PLATFORMS

data/lib/qprocessor/beanstalk_message.rb

@@ -0,0 +1,24 @@
+require "qprocessor/message"
+
+module QProcessor
+  class BeanstalkMessage < Message
+    def initialize(source)
+      super(source)
+    end
+
+    # Disposes of the underlying job, deleting it from Beanstalk.
+    def dispose
+      source.delete
+    end
+
+    # Retrieve the message identifier.
+    def id
+      source.id
+    end
+
+    # Releases the job back to Beanstalk.
+    def release
+      source.release
+    end
+  end
+end

data/lib/qprocessor/beanstalk_source.rb

@@ -0,0 +1,46 @@
+require "beaneater"
+require "qprocessor/beanstalk_message"
+
+module QProcessor
+  class BeanstalkSource
+    # A constant for a regular expression to match Beanstalk URLs
+    BEANSTALK_URL_PATTERN = /^beanstalk:\/\/([^:\/]+)[:]?([\d]*)[\/]?(.*)$/
+
+    # The default Beanstalk tube name.
+    DEFAULT_TUBE_NAME = "default"
+
+    def initialize
+      @name = "Beanstalk Source"
+    end
+    attr_reader :name
+
+    # This method blocks until a job is available, at which points it is
+    # returned.
+    def get
+      job     = connection.reserve
+      message = QProcessor::BeanstalkMessage.new(job)
+      yield message if block_given?
+      message
+    end
+
+  private
+
+    def beanstalk_url
+      @beanstalk_url ||= ENV["BEANSTALK_URL"]
+    end
+
+    def connection
+      if !@connection
+        match = BEANSTALK_URL_PATTERN.match(beanstalk_url)
+        raise QProcessor::Error("Invalid Beanstalk queue URL '#{beanstalk_url}' specified.") if !match
+        beanstalk   = Beaneater.new("#{match[1]}#{match[2] == "" ? "" : ":#{match[2]}"}")
+        @connection = beanstalk.tubes[match[3] != "" ? match[3] : DEFAULT_TUBE_NAME]
+      end
+      @connection
+    end
+
+    def tube_name
+      ENV["BEANSTALK_TUBE_NAME"] || DEFAULT_TUBE_NAME
+    end
+  end
+end

data/lib/qprocessor/message.rb

@@ -0,0 +1,37 @@
+module QProcessor
+  # This is a class representing a message or job pulled from a queue with the
+  # intention of this class providing a unified interface around different
+  # library classes.
+  class Message
+    def initialize(source)
+      @source = source
+    end
+    attr_reader :source
+
+    # Retrieves the content of the message/job as a String.
+    def body
+      @source.body
+    end
+
+    # This method should do whatever is necessary to clean up a message/job such
+    # that the underlying queuing mechanism will no longer deliver it. This is
+    # a 'do nothing' implementation that can be overridden by derived classes to
+    # provide for this functionality.
+    def dispose
+    end
+
+    # Fetches a unique identifer for the message/job. This default implementation
+    # raises an exception. Derived classes can override this to provide the
+    # required functionality.
+    def id
+      raise "The #{self.class.name} class does not override the #id() method."
+    end
+
+    # This method should do whatever is necessary to return a message/job to the
+    # underlying queuing mechanism such that it becomes available again for
+    # delivery. This is a 'do nothing' implementation that can be overridden by
+    # derived classes to provide for this functionality.
+    def release
+    end
+  end
+end

data/lib/qprocessor/processor.rb

@@ -1,27 +1,21 @@
 require "beaneater"
 require "logger"
+require "qprocessor/sources"
 
 module QProcessor
   # This class encapsulates a lot of the functionality around creating a queue
   # processor that pulls work from a queue and passes it to an instance of a
   # specified class for 'handling'.
   class Processor
-    # A constant for a regular expression to match Beanstalk URLs
-    BEANSTALK_URL_PATTERN = /^beanstalk:\/\/([^:\/]+)[:]?([\d]*)[\/]?(.*)$/
-
     # A constant with the name of the default job parser class.
     DEFAULT_PARSER_CLASS = "QProcessor::YAMLParser"
 
-    # The default Beanstalk tube name.
-    DEFAULT_TUBE_NAME = "default"
-
     # A constant containing the maximum permitted setting for the inter-error sleep interval.
     MAX_ERROR_RESTART_INTERVAL = 32
 
     # Constructor for the Processor class.
     #
     # @param [String] The name assigned to the processor, primarily used for logging.
-    # @param [String] The URL to be used when connecting to the queue system.
     # @param [Hash] A Hash of the class names used by the processor. There are two
     #               keys recognised in this Hash - :parser and :processor. The
     #               :parser class should be the fully qualified name of the class
@@ -30,12 +24,11 @@ module QProcessor
     #               process jobs.
     # @param [Hash] A Hash of additional settings that will be used by the processor.
     #               Valid keys include :logger and :reuse_processor.
-    def initialize(name, url, class_names, settings={})
+    def initialize(name, class_names, settings={})
       @instance        = nil
       @name            = name
       @parser_class    = get_class!(class_names.fetch(:parser, DEFAULT_PARSER_CLASS))
       @processor_class = get_class!(class_names[:processor])
-      @queue_url       = url
       @settings        = {}.merge(settings)
       @terminate       = false
     end
@@ -44,15 +37,20 @@ module QProcessor
     # Starts processing of the queue. This method will not return to the caller
     # as it will run a loop processing the queue jobs as they become available.
     def start
-      queue    = connect
+      queue    = QProcessor::Sources.manufacture!
       interval = 0
       while !@terminate
         begin
           logger.debug("The '#{name}' queue processor is listening for jobs.")
-          queue.reserve do|job|
-            logger.debug "The '#{name}' queue processor received job id #{job.id}."
-            handle(job)
-            interval = 0
+          queue.get do|message|
+            begin
+              logger.debug "The '#{name}' queue processor received message id #{message.id}."
+              handle(message)
+              interval = 0
+            rescue => error
+              message.release
+              raise
+            end
           end
         rescue => error
           logger.error "The '#{name}' queue processor caught an exception.\nType: #{error.class.name}\n"\
@@ -66,21 +64,6 @@ module QProcessor
 
   private
 
-    def beanstalk_connect
-      match = BEANSTALK_URL_PATTERN.match(queue_url)
-      raise QProcessor::Error("Invalid Beanstalk queue URL '#{queue_url}' specified.") if !match
-      beanstalk = Beaneater.new("#{match[1]}#{match[2] == "" ? "" : ":#{match[2]}"}")
-      beanstalk.tubes[match[3] != "" ? match[3] : DEFAULT_TUBE_NAME]
-    end
-
-    def connect
-      if queue_url[0, 9] == "beanstalk"
-        beanstalk_connect
-      else
-        raise QProcessor::Error.new("Unrecognised queue URL '#{queue_url}' encountered")
-      end
-    end
-
     def get_class(name)
       name.split("::").reduce(Object) {|t, e| (!t.nil? ? t.const_get(e) : nil)}
     end

data/lib/qprocessor/sources.rb

@@ -0,0 +1,22 @@
+require "qprocessor/beanstalk_source"
+require "qprocessor/sqs_source"
+
+module QProcessor
+  module Sources
+    # This method checks environment variables to see if any are set such that
+    # a queue source can be generated from them. When checking environment
+    # variables Beanstalk has highest priority, followed by SQS. If no matching
+    # environment variables are found then an exception will be raised. For
+    # Beanstalk the BEANSTALK_URL environment variable is checked for. For SQS
+    # the SQS_QUEUE_NAME environment variable is check for.
+    def self.manufacture!
+      if !ENV["BEANSTALK_URL"].nil?
+        QProcessor::BeanstalkSource.new
+      elsif !ENV["SQS_QUEUE_NAME"].nil?
+        QProcessor::SQSSource.new
+      else
+        raise "Unable to determine source queue type from environment settings."
+      end
+    end
+  end
+end

data/lib/qprocessor/sqs_message.rb

@@ -0,0 +1,21 @@
+require "qprocessor/message"
+
+module QProcessor
+  class SQSMessage < Message
+    def initialize(source, client, queue_url)
+      super(source)
+      @client    = client
+      @queue_url = queue_url
+    end
+
+    # Deletes a message from the source queue.
+    def dispose
+      @client.delete_message(queue_url: @queue_url, receipt_handle: source.receipt_handle)
+    end
+
+    # Retrieves the message identifier.
+    def id
+      source.message_id
+    end
+  end
+end

data/lib/qprocessor/sqs_source.rb

@@ -0,0 +1,99 @@
+require "aws-sdk-sqs"
+require "aws-sdk-sts"
+require "qprocessor/sqs_message"
+require "pp"
+
+module QProcessor
+  class SQSSource
+    # A constant for the maximum wait for message timeout to be used by the
+    # source.
+    MAX_MESSAGE_WAIT = 3
+
+    def initialize
+      @name = "SQS Source"
+    end
+    attr_reader :name
+
+    # This method blocks until a job is available, at which points it is
+    # returned.
+    def get
+      message = nil
+      client  = sqs_client
+      url     = sqs_queue_url(sqs_queue_name)
+      while message.nil?
+        result  = client.receive_message(max_number_of_messages: 1,
+                                         queue_url:              url,
+                                         wait_time_seconds:      MAX_MESSAGE_WAIT)
+        message = QProcessor::SQSMessage.new(result.messages[0], client, url) if !result.messages.empty?
+      end
+      yield message if block_given?
+      message
+    end
+
+  private
+
+    # Attempts to fetch the AWS access key from environment variables, raising
+    # an exception if not found.
+    def aws_access_key
+      @aws_access_key ||= ENV["AWS_ACCESS_KEY"]
+      raise "The AWS_ACCESS_KEY environment variable has not been set." if @aws_access_key.nil?
+      @aws_access_key
+    end
+
+    # Creates a Aws::Credentials instance from AWS settings read from the
+    # environment. An exception will be raised if any of these settings is
+    # not present.
+    def aws_credentials
+      Aws::Credentials.new(aws_access_key, aws_secret_key)
+    end
+
+    # Attempts to fetch the AWS region from environment variables, raising an
+    # exception if not found.
+    def aws_region
+      @aws_region ||= ENV["AWS_REGION"]
+      raise "The AWS_REGION environment variable has not been set." if @aws_region.nil?
+      @aws_region
+    end
+
+    # Attempts to fetch the AWS secret key from environment variables, raising
+    # an exception if not found.
+    def aws_secret_key
+      @aws_secret_key ||= ENV["AWS_SECRET_KEY"]
+      raise "The AWS_SECRET_KEY environment variable has not been set." if @aws_secret_key.nil?
+      @aws_secret_key
+    end
+
+    def connection
+      if !@connection
+        url = beanstalk_url
+        raise "No BEANSTALK_URL environment variable set." if url.nil?
+        @connection ||= Beaneater.new(url)
+      end
+      @connection
+    end
+
+    # Fetches a Aws::SQS::Client instance using AWS credentials and region
+    # details that are read from environment settings. If any of these settings
+    # are missing an exception will be raised.
+    def sqs_client
+      Aws::SQS::Client.new(credentials: aws_credentials, region: aws_region)
+    end
+
+    # Attempts to fetch the SQS queue name from environment variables, raising
+    # an exception if not found.
+    def sqs_queue_name
+      @sqs_queue_name ||= ENV["SQS_QUEUE_NAME"]
+      raise "The SQS_QUEUE_NAME environment variable has not been set." if @sqs_queue_name.nil?
+      @sqs_queue_name
+    end
+
+    # Generates an URL for use with a specific AWS SQS queue.
+    def sqs_queue_url(queue_name)
+      if !@sqs_queue_url
+        client = Aws::STS::Client.new(credentials: aws_credentials, region: aws_region)
+        @sqs_queue_url = "https://sqs.#{aws_region}.amazonaws.com/#{client.get_caller_identity.account}/#{queue_name}"
+      end
+      @sqs_queue_url
+    end
+  end
+end

data/lib/qprocessor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module QProcessor
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/qprocessor.gemspec

@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_dependency "aws-sdk-sqs", "~> 1.49"
+  spec.add_dependency "aws-sdk-sts", "~> 1.5"
   spec.add_dependency "beaneater", "~> 1.1"
 
   # For more information and examples about making a new gem, checkout our

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: qprocessor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Peter Wood
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-01-11 00:00:00.000000000 Z
+date: 2022-01-14 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.49'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.49'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: beaneater
   requirement: !ruby/object:Gem::Requirement
@@ -42,8 +70,14 @@ files:
 - bin/console
 - bin/setup
 - lib/qprocessor.rb
+- lib/qprocessor/beanstalk_message.rb
+- lib/qprocessor/beanstalk_source.rb
 - lib/qprocessor/json_parser.rb
+- lib/qprocessor/message.rb
 - lib/qprocessor/processor.rb
+- lib/qprocessor/sources.rb
+- lib/qprocessor/sqs_message.rb
+- lib/qprocessor/sqs_source.rb
 - lib/qprocessor/version.rb
 - lib/qprocessor/yaml_parser.rb
 - qprocessor.gemspec