qprocessor 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: acc989a7981c73a9503e369c54b4c912f3dab1ca073c33d49a38c379db2c0b7b
-  data.tar.gz: c408d6c67f41b2e0da1992aa8226d3b1bdd3e24c6ac9eda8bb91ed4010061af3
+  metadata.gz: 7e49f396b457d247ba22300439d134f4d663696f445a32dc36451fae46566602
+  data.tar.gz: 5ef52af86d0a282e5e633c156f7fcf3d1c08ccbb079cd0c5cb5f98eec3ab33ef
 SHA512:
-  metadata.gz: 62068b04ecf60602b1094106c9fc0700659af07aa779b8f4b89e608a81f95e60fd29ef93257eee67f2cd0e1fa271bf25717017beed998b3b133c45a86f2e3d5d
-  data.tar.gz: ddeb4a8105387ec43a35a07edae349f6952a71e9b15d64b6e03b6bf0414bccfa57523067984610c36ef8422ff745358ac56ccd20e3c77c469e5006c30df6ac44
+  metadata.gz: f4711089a7f42c713698c5183e56b3dbc287f4f25f3fb3197dfe5eb55e0f6dda809f558fc6e124f6b1a3b077045715a7b763518b08878ded26303e2d2b6258ff
+  data.tar.gz: 026c0bdc955bfcb61201b29453bbe2db5f1685b5273df674ac7b439551a9fe7b3e176cf26a71e5d520aa49af0c13a98de9b894221b1fe08d4d98ef8399edba0f

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile.lock

@@ -1,17 +1,18 @@
 PATH
   remote: .
   specs:
-    qprocessor (0.2.0)
+    qprocessor (0.3.0)
       aws-sdk-sqs (~> 1.49)
       aws-sdk-sts (~> 1.5)
       beaneater (~> 1.1)
+      ox (~> 2.14)
 
 GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.2.0)
-    aws-partitions (1.548.0)
-    aws-sdk-core (3.125.3)
+    aws-partitions (1.550.0)
+    aws-sdk-core (3.125.5)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
@@ -25,8 +26,23 @@ GEM
     aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     beaneater (1.1.1)
+    diff-lcs (1.5.0)
     jmespath (1.5.0)
+    ox (2.14.6)
     rake (13.0.6)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
 
 PLATFORMS
   x86_64-linux
@@ -34,6 +50,7 @@ PLATFORMS
 DEPENDENCIES
   qprocessor!
   rake (~> 13.0)
+  rspec
 
 BUNDLED WITH
    2.2.3

data/README.md

@@ -1,11 +1,11 @@
 # QProcessor
 
-A simple library that wraps the processing of elements pulled from a queue. Initial
-implementation will work with Beanstalkd, though I might expand it to cover other
-types of queue later. The idea behind this gem is to allow for the easy creation
-of an application as a service, where the services job is to pull items from a
-queue and process them. The library code is intended to wrap the parts of the code
-that would be common and repeated if you made individual service applications.
+A simple library that wraps the processing of elements pulled from a queue. The
+current implementation supports the Beanstalkd and SQS queuing platforms. The
+idea behind this gem is to allow for the easy creation of an 'application as a
+service', where the services jobs is to pull items from a queue and process them.
+The library code is intended to wrap the parts of the code that would be common
+and repeated if you made individual service applications.
 
 ## Installation
 
@@ -25,7 +25,83 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+There are two things that you need to do to make use of this library. The first
+is to create a processor class and the second is to assemble the service
+execution functionality.
+
+### The Processor Class
+
+The processor class is a standard Ruby class that obeys the following rules...
+
+ 1. It takes a Hash of settings for the constructor parameter. These will be
+    passed down from the service when the class is instantiated.
+
+ 2. It implements a ``process()`` method that takes a single parameter. This
+    parameter will be the message that was obtained from the queue. The
+    message has a ``body()`` accessor method that grants access to a ``String``
+    containing the message contents that are to be processed. The ``process()``
+    method should conduct whatever work is needed based on the message. If
+    processing of the message is complete then the ``process()`` method should
+    call the ``dispose()`` method of the message parameter received as this
+    informs the queuing mechanism that the message has been completely dealt
+    with and should not be issued to any other requesters.
+
+### The Service
+
+The service should follow this general structure...
+
+```ruby
+require "qprocessor"
+require "processor_class"
+
+begin
+  processor = QProcessor::Processor.new(ProcessorClass, logger: logger)
+  processor.start
+rescue => error
+  STDERR.puts "ERROR: #{error}\n#{error.backtrace.join("\n")}"
+end
+```
+
+First thing this does is require in the qprocessor library and then the processor
+class (as outlined in the previous section). Next it creates an instance of the
+``QProcessor::Processor`` class, passing it at least two parameters. The first
+parameter is a reference to the Processor class that will be used by this
+instance to handle entries read from the queue. This should be an instance of
+the ``Class`` class.
+
+The constructor accepts additional parameters in the form of a ``Hash`` of
+settings. These settings will be passed down to the processor class whenever
+it gets instantiated but they may also be used by the queue processor
+functionality itself. For example, if the settings include  the key
+``reuse_processor``, then the processor class will only be instantiated once
+and then re-used for all messages. You can use this settings ``Hash`` to pass
+value to the processing class that are reusable resources or that might include
+configuration.
+
+## Configuration
+
+You need to provide the qprocessor service with details for the queue that it
+will be working with. Currently supported queueing platforms include AQWS SQS
+and Beanstalkd. Configuring which platform and which queue is used is done
+through environment settings are outlined below...
+
+### Beanstalkd
+
+To use a Beanstalkd queue (tube) then you must set the ``BEANSTALK_URL``
+environment setting. An example of this might be...
+
+    ``beanstalk://host:port/tube_name``
+
+### AWS SQS
+
+To use the AWS SQS queuing platform you must set the ``SQS_QUEUE_NAME``
+environment setting. Note that the Beanstalk configuration, if present, will
+be used in preference to the SQS one - so don't configure both settings on
+the same processor. The ``SQS_QUEUE_NAME`` environment variable can just be
+the name of the SQS queue that will be used. The AWS SQS processor also needs
+the ``AWS_ACCESS_KEY``, ``AWS_REGION`` and ``AWS_SECRET_KEY`` environment
+variables set to appropriate values for use in getting access to the SQS
+queue specified.
 
 ## Contributing
 

data/lib/qprocessor/processor.rb

@@ -7,28 +7,20 @@ module QProcessor
   # processor that pulls work from a queue and passes it to an instance of a
   # specified class for 'handling'.
   class Processor
-    # A constant with the name of the default job parser class.
-    DEFAULT_PARSER_CLASS = "QProcessor::YAMLParser"
-
     # A constant containing the maximum permitted setting for the inter-error sleep interval.
     MAX_ERROR_RESTART_INTERVAL = 32
 
     # Constructor for the Processor class.
     #
-    # @param [String] The name assigned to the processor, primarily used for logging.
-    # @param [Hash] A Hash of the class names used by the processor. There are two
-    #               keys recognised in this Hash - :parser and :processor. The
-    #               :parser class should be the fully qualified name of the class
-    #               class that can be used to parse job content. The :processor
-    #               class should be the fully qualified name of the class that will
-    #               process jobs.
+    # @param [Class] A reference to the class that will used to process jobs.
     # @param [Hash] A Hash of additional settings that will be used by the processor.
-    #               Valid keys include :logger and :reuse_processor.
-    def initialize(name, class_names, settings={})
+    #               These will get passed to the processor class when it is
+    #               instantiated and provides a mechansim for passing in elements
+    #               such as configuration or settings.
+    def initialize(processor, settings={})
       @instance        = nil
-      @name            = name
-      @parser_class    = get_class!(class_names.fetch(:parser, DEFAULT_PARSER_CLASS))
-      @processor_class = get_class!(class_names[:processor])
+      @name            = processor.class.name
+      @processor_class = processor
       @settings        = {}.merge(settings)
       @terminate       = false
     end
@@ -42,16 +34,7 @@ module QProcessor
       while !@terminate
         begin
           logger.debug("The '#{name}' queue processor is listening for jobs.")
-          queue.get do|message|
-            begin
-              logger.debug "The '#{name}' queue processor received message id #{message.id}."
-              handle(message)
-              interval = 0
-            rescue => error
-              message.release
-              raise
-            end
-          end
+          queue.get {|message| handle(message)}
         rescue => error
           logger.error "The '#{name}' queue processor caught an exception.\nType: #{error.class.name}\n"\
                        "Message: #{error}\nStack Trace:\n#{error.backtrace.join("\n")}"
@@ -62,28 +45,38 @@ module QProcessor
       end
     end
 
-  private
-
-    def get_class(name)
-      name.split("::").reduce(Object) {|t, e| (!t.nil? ? t.const_get(e) : nil)}
+    def handle(message)
+      begin
+        logger.debug "The '#{name}' queue processor received message id #{message.id}."
+        process(message)
+        message.dispose
+        interval = 0
+      rescue => error
+        message.release
+        raise
+      end
     end
 
-    def get_class!(name)
-      result = get_class(name)
-      raise QProcessor::Error.new("Unable to locate the '#{class_name}' class.") if result.nil?
-      result
-    end
+  private
 
-    def handle(job)
-      @instance = @processor_class.new(parser: @parser_class.new(logger: logger),
-                                       logger: logger) if @instance.nil?
+    def process(job)
+      @instance = @processor_class.new(settings) if @instance.nil?
       @instance.process(job)
       @instance = nil if !settings.fetch(:reuse_processor, true)
     end
 
     def logger
-      settings[:logger] = Logger.new(STDOUT) if !settings.include?(:logger) || !settings[:logger]
+      if !settings.include?(:logger) || !settings[:logger]
+        settings[:logger] = Logger.new(STDOUT)
+        settings[:logger].level = logging_level
+      end
       settings[:logger]
     end
+
+    def logging_level
+      name = ENV["LOGGING_LEVEL"].to_s
+      name = "INFO" if name == "" || !Logger.const_defined?(name)
+      Logger.const_get(name)
+    end
   end
 end

data/lib/qprocessor/sqs_source.rb

@@ -63,15 +63,6 @@ module QProcessor
       @aws_secret_key
     end
 
-    def connection
-      if !@connection
-        url = beanstalk_url
-        raise "No BEANSTALK_URL environment variable set." if url.nil?
-        @connection ||= Beaneater.new(url)
-      end
-      @connection
-    end
-
     # Fetches a Aws::SQS::Client instance using AWS credentials and region
     # details that are read from environment settings. If any of these settings
     # are missing an exception will be raised.

data/lib/qprocessor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module QProcessor
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/qprocessor.gemspec

@@ -32,6 +32,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk-sqs", "~> 1.49"
   spec.add_dependency "aws-sdk-sts", "~> 1.5"
   spec.add_dependency "beaneater", "~> 1.1"
+  spec.add_dependency "ox", "~> 2.14"
+
+  # Development dependency.
+  spec.add_development_dependency "rspec"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qprocessor
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Peter Wood
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-01-14 00:00:00.000000000 Z
+date: 2022-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: ox
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A library to assist in the creation of queue processing 'application
   as a service' entities.
 email:
@@ -61,6 +89,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock