qprocessor 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8e127381632515acbf030a9eca5338928a298187b155950102a33bff338af4fc
4
- data.tar.gz: 767a5e859b1de2fcc61463b2b0c2d350a4ca7dbffc7ede0d352005fab68707cd
3
+ metadata.gz: acc989a7981c73a9503e369c54b4c912f3dab1ca073c33d49a38c379db2c0b7b
4
+ data.tar.gz: c408d6c67f41b2e0da1992aa8226d3b1bdd3e24c6ac9eda8bb91ed4010061af3
5
5
  SHA512:
6
- metadata.gz: de8a0421e4fd44bb25badc482f8aa1a6b160f0d73594e3d3a689f9e538e57bcd37b0191d168206546776cc7a55a6e054e495db3a0bbeb0f90561c9087e4fe19f
7
- data.tar.gz: fbbbd2241b98d85028964409991fcf87ddde61cfecaddc4ca99677717c828a2a1d6604e4bb3232f438aebc3ee742f0dbcb41520c3a7afe55e4ef18f38333d8fb
6
+ metadata.gz: 62068b04ecf60602b1094106c9fc0700659af07aa779b8f4b89e608a81f95e60fd29ef93257eee67f2cd0e1fa271bf25717017beed998b3b133c45a86f2e3d5d
7
+ data.tar.gz: ddeb4a8105387ec43a35a07edae349f6952a71e9b15d64b6e03b6bf0414bccfa57523067984610c36ef8422ff745358ac56ccd20e3c77c469e5006c30df6ac44
data/Gemfile.lock CHANGED
@@ -1,13 +1,31 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- qprocessor (0.1.0)
4
+ qprocessor (0.2.0)
5
+ aws-sdk-sqs (~> 1.49)
6
+ aws-sdk-sts (~> 1.5)
5
7
  beaneater (~> 1.1)
6
8
 
7
9
  GEM
8
10
  remote: https://rubygems.org/
9
11
  specs:
12
+ aws-eventstream (1.2.0)
13
+ aws-partitions (1.548.0)
14
+ aws-sdk-core (3.125.3)
15
+ aws-eventstream (~> 1, >= 1.0.2)
16
+ aws-partitions (~> 1, >= 1.525.0)
17
+ aws-sigv4 (~> 1.1)
18
+ jmespath (~> 1.0)
19
+ aws-sdk-sqs (1.49.0)
20
+ aws-sdk-core (~> 3, >= 3.125.0)
21
+ aws-sigv4 (~> 1.1)
22
+ aws-sdk-sts (1.5.0)
23
+ aws-sdk-core (~> 3, >= 3.110.0)
24
+ aws-sigv4 (~> 1.1)
25
+ aws-sigv4 (1.4.0)
26
+ aws-eventstream (~> 1, >= 1.0.2)
10
27
  beaneater (1.1.1)
28
+ jmespath (1.5.0)
11
29
  rake (13.0.6)
12
30
 
13
31
  PLATFORMS
@@ -0,0 +1,24 @@
1
+ require "qprocessor/message"
2
+
3
+ module QProcessor
4
+ class BeanstalkMessage < Message
5
+ def initialize(source)
6
+ super(source)
7
+ end
8
+
9
+ # Disposes of the underlying job, deleting it from Beanstalk.
10
+ def dispose
11
+ source.delete
12
+ end
13
+
14
+ # Retrieve the message identifier.
15
+ def id
16
+ source.id
17
+ end
18
+
19
+ # Releases the job back to Beanstalk.
20
+ def release
21
+ source.release
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,46 @@
1
+ require "beaneater"
2
+ require "qprocessor/beanstalk_message"
3
+
4
+ module QProcessor
5
+ class BeanstalkSource
6
+ # A constant for a regular expression to match Beanstalk URLs
7
+ BEANSTALK_URL_PATTERN = /^beanstalk:\/\/([^:\/]+)[:]?([\d]*)[\/]?(.*)$/
8
+
9
+ # The default Beanstalk tube name.
10
+ DEFAULT_TUBE_NAME = "default"
11
+
12
+ def initialize
13
+ @name = "Beanstalk Source"
14
+ end
15
+ attr_reader :name
16
+
17
+ # This method blocks until a job is available, at which points it is
18
+ # returned.
19
+ def get
20
+ job = connection.reserve
21
+ message = QProcessor::BeanstalkMessage.new(job)
22
+ yield message if block_given?
23
+ message
24
+ end
25
+
26
+ private
27
+
28
+ def beanstalk_url
29
+ @beanstalk_url ||= ENV["BEANSTALK_URL"]
30
+ end
31
+
32
+ def connection
33
+ if !@connection
34
+ match = BEANSTALK_URL_PATTERN.match(beanstalk_url)
35
+ raise QProcessor::Error("Invalid Beanstalk queue URL '#{beanstalk_url}' specified.") if !match
36
+ beanstalk = Beaneater.new("#{match[1]}#{match[2] == "" ? "" : ":#{match[2]}"}")
37
+ @connection = beanstalk.tubes[match[3] != "" ? match[3] : DEFAULT_TUBE_NAME]
38
+ end
39
+ @connection
40
+ end
41
+
42
+ def tube_name
43
+ ENV["BEANSTALK_TUBE_NAME"] || DEFAULT_TUBE_NAME
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,37 @@
1
+ module QProcessor
2
+ # This is a class representing a message or job pulled from a queue with the
3
+ # intention of this class providing a unified interface around different
4
+ # library classes.
5
+ class Message
6
+ def initialize(source)
7
+ @source = source
8
+ end
9
+ attr_reader :source
10
+
11
+ # Retrieves the content of the message/job as a String.
12
+ def body
13
+ @source.body
14
+ end
15
+
16
+ # This method should do whatever is necessary to clean up a message/job such
17
+ # that the underlying queuing mechanism will no longer deliver it. This is
18
+ # a 'do nothing' implementation that can be overridden by derived classes to
19
+ # provide for this functionality.
20
+ def dispose
21
+ end
22
+
23
+ # Fetches a unique identifer for the message/job. This default implementation
24
+ # raises an exception. Derived classes can override this to provide the
25
+ # required functionality.
26
+ def id
27
+ raise "The #{self.class.name} class does not override the #id() method."
28
+ end
29
+
30
+ # This method should do whatever is necessary to return a message/job to the
31
+ # underlying queuing mechanism such that it becomes available again for
32
+ # delivery. This is a 'do nothing' implementation that can be overridden by
33
+ # derived classes to provide for this functionality.
34
+ def release
35
+ end
36
+ end
37
+ end
@@ -1,27 +1,21 @@
1
1
  require "beaneater"
2
2
  require "logger"
3
+ require "qprocessor/sources"
3
4
 
4
5
  module QProcessor
5
6
  # This class encapsulates a lot of the functionality around creating a queue
6
7
  # processor that pulls work from a queue and passes it to an instance of a
7
8
  # specified class for 'handling'.
8
9
  class Processor
9
- # A constant for a regular expression to match Beanstalk URLs
10
- BEANSTALK_URL_PATTERN = /^beanstalk:\/\/([^:\/]+)[:]?([\d]*)[\/]?(.*)$/
11
-
12
10
  # A constant with the name of the default job parser class.
13
11
  DEFAULT_PARSER_CLASS = "QProcessor::YAMLParser"
14
12
 
15
- # The default Beanstalk tube name.
16
- DEFAULT_TUBE_NAME = "default"
17
-
18
13
  # A constant containing the maximum permitted setting for the inter-error sleep interval.
19
14
  MAX_ERROR_RESTART_INTERVAL = 32
20
15
 
21
16
  # Constructor for the Processor class.
22
17
  #
23
18
  # @param [String] The name assigned to the processor, primarily used for logging.
24
- # @param [String] The URL to be used when connecting to the queue system.
25
19
  # @param [Hash] A Hash of the class names used by the processor. There are two
26
20
  # keys recognised in this Hash - :parser and :processor. The
27
21
  # :parser class should be the fully qualified name of the class
@@ -30,12 +24,11 @@ module QProcessor
30
24
  # process jobs.
31
25
  # @param [Hash] A Hash of additional settings that will be used by the processor.
32
26
  # Valid keys include :logger and :reuse_processor.
33
- def initialize(name, url, class_names, settings={})
27
+ def initialize(name, class_names, settings={})
34
28
  @instance = nil
35
29
  @name = name
36
30
  @parser_class = get_class!(class_names.fetch(:parser, DEFAULT_PARSER_CLASS))
37
31
  @processor_class = get_class!(class_names[:processor])
38
- @queue_url = url
39
32
  @settings = {}.merge(settings)
40
33
  @terminate = false
41
34
  end
@@ -44,15 +37,20 @@ module QProcessor
44
37
  # Starts processing of the queue. This method will not return to the caller
45
38
  # as it will run a loop processing the queue jobs as they become available.
46
39
  def start
47
- queue = connect
40
+ queue = QProcessor::Sources.manufacture!
48
41
  interval = 0
49
42
  while !@terminate
50
43
  begin
51
44
  logger.debug("The '#{name}' queue processor is listening for jobs.")
52
- queue.reserve do|job|
53
- logger.debug "The '#{name}' queue processor received job id #{job.id}."
54
- handle(job)
55
- interval = 0
45
+ queue.get do|message|
46
+ begin
47
+ logger.debug "The '#{name}' queue processor received message id #{message.id}."
48
+ handle(message)
49
+ interval = 0
50
+ rescue => error
51
+ message.release
52
+ raise
53
+ end
56
54
  end
57
55
  rescue => error
58
56
  logger.error "The '#{name}' queue processor caught an exception.\nType: #{error.class.name}\n"\
@@ -66,21 +64,6 @@ module QProcessor
66
64
 
67
65
  private
68
66
 
69
- def beanstalk_connect
70
- match = BEANSTALK_URL_PATTERN.match(queue_url)
71
- raise QProcessor::Error("Invalid Beanstalk queue URL '#{queue_url}' specified.") if !match
72
- beanstalk = Beaneater.new("#{match[1]}#{match[2] == "" ? "" : ":#{match[2]}"}")
73
- beanstalk.tubes[match[3] != "" ? match[3] : DEFAULT_TUBE_NAME]
74
- end
75
-
76
- def connect
77
- if queue_url[0, 9] == "beanstalk"
78
- beanstalk_connect
79
- else
80
- raise QProcessor::Error.new("Unrecognised queue URL '#{queue_url}' encountered")
81
- end
82
- end
83
-
84
67
  def get_class(name)
85
68
  name.split("::").reduce(Object) {|t, e| (!t.nil? ? t.const_get(e) : nil)}
86
69
  end
@@ -0,0 +1,22 @@
1
+ require "qprocessor/beanstalk_source"
2
+ require "qprocessor/sqs_source"
3
+
4
+ module QProcessor
5
+ module Sources
6
+ # This method checks environment variables to see if any are set such that
7
+ # a queue source can be generated from them. When checking environment
8
+ # variables Beanstalk has highest priority, followed by SQS. If no matching
9
+ # environment variables are found then an exception will be raised. For
10
+ # Beanstalk the BEANSTALK_URL environment variable is checked for. For SQS
11
+ # the SQS_QUEUE_NAME environment variable is check for.
12
+ def self.manufacture!
13
+ if !ENV["BEANSTALK_URL"].nil?
14
+ QProcessor::BeanstalkSource.new
15
+ elsif !ENV["SQS_QUEUE_NAME"].nil?
16
+ QProcessor::SQSSource.new
17
+ else
18
+ raise "Unable to determine source queue type from environment settings."
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,21 @@
1
+ require "qprocessor/message"
2
+
3
+ module QProcessor
4
+ class SQSMessage < Message
5
+ def initialize(source, client, queue_url)
6
+ super(source)
7
+ @client = client
8
+ @queue_url = queue_url
9
+ end
10
+
11
+ # Deletes a message from the source queue.
12
+ def dispose
13
+ @client.delete_message(queue_url: @queue_url, receipt_handle: source.receipt_handle)
14
+ end
15
+
16
+ # Retrieves the message identifier.
17
+ def id
18
+ source.message_id
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,99 @@
1
+ require "aws-sdk-sqs"
2
+ require "aws-sdk-sts"
3
+ require "qprocessor/sqs_message"
4
+ require "pp"
5
+
6
+ module QProcessor
7
+ class SQSSource
8
+ # A constant for the maximum wait for message timeout to be used by the
9
+ # source.
10
+ MAX_MESSAGE_WAIT = 3
11
+
12
+ def initialize
13
+ @name = "SQS Source"
14
+ end
15
+ attr_reader :name
16
+
17
+ # This method blocks until a job is available, at which points it is
18
+ # returned.
19
+ def get
20
+ message = nil
21
+ client = sqs_client
22
+ url = sqs_queue_url(sqs_queue_name)
23
+ while message.nil?
24
+ result = client.receive_message(max_number_of_messages: 1,
25
+ queue_url: url,
26
+ wait_time_seconds: MAX_MESSAGE_WAIT)
27
+ message = QProcessor::SQSMessage.new(result.messages[0], client, url) if !result.messages.empty?
28
+ end
29
+ yield message if block_given?
30
+ message
31
+ end
32
+
33
+ private
34
+
35
+ # Attempts to fetch the AWS access key from environment variables, raising
36
+ # an exception if not found.
37
+ def aws_access_key
38
+ @aws_access_key ||= ENV["AWS_ACCESS_KEY"]
39
+ raise "The AWS_ACCESS_KEY environment variable has not been set." if @aws_access_key.nil?
40
+ @aws_access_key
41
+ end
42
+
43
+ # Creates a Aws::Credentials instance from AWS settings read from the
44
+ # environment. An exception will be raised if any of these settings is
45
+ # not present.
46
+ def aws_credentials
47
+ Aws::Credentials.new(aws_access_key, aws_secret_key)
48
+ end
49
+
50
+ # Attempts to fetch the AWS region from environment variables, raising an
51
+ # exception if not found.
52
+ def aws_region
53
+ @aws_region ||= ENV["AWS_REGION"]
54
+ raise "The AWS_REGION environment variable has not been set." if @aws_region.nil?
55
+ @aws_region
56
+ end
57
+
58
+ # Attempts to fetch the AWS secret key from environment variables, raising
59
+ # an exception if not found.
60
+ def aws_secret_key
61
+ @aws_secret_key ||= ENV["AWS_SECRET_KEY"]
62
+ raise "The AWS_SECRET_KEY environment variable has not been set." if @aws_secret_key.nil?
63
+ @aws_secret_key
64
+ end
65
+
66
+ def connection
67
+ if !@connection
68
+ url = beanstalk_url
69
+ raise "No BEANSTALK_URL environment variable set." if url.nil?
70
+ @connection ||= Beaneater.new(url)
71
+ end
72
+ @connection
73
+ end
74
+
75
+ # Fetches a Aws::SQS::Client instance using AWS credentials and region
76
+ # details that are read from environment settings. If any of these settings
77
+ # are missing an exception will be raised.
78
+ def sqs_client
79
+ Aws::SQS::Client.new(credentials: aws_credentials, region: aws_region)
80
+ end
81
+
82
+ # Attempts to fetch the SQS queue name from environment variables, raising
83
+ # an exception if not found.
84
+ def sqs_queue_name
85
+ @sqs_queue_name ||= ENV["SQS_QUEUE_NAME"]
86
+ raise "The SQS_QUEUE_NAME environment variable has not been set." if @sqs_queue_name.nil?
87
+ @sqs_queue_name
88
+ end
89
+
90
+ # Generates an URL for use with a specific AWS SQS queue.
91
+ def sqs_queue_url(queue_name)
92
+ if !@sqs_queue_url
93
+ client = Aws::STS::Client.new(credentials: aws_credentials, region: aws_region)
94
+ @sqs_queue_url = "https://sqs.#{aws_region}.amazonaws.com/#{client.get_caller_identity.account}/#{queue_name}"
95
+ end
96
+ @sqs_queue_url
97
+ end
98
+ end
99
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module QProcessor
4
- VERSION = "0.1.0"
4
+ VERSION = "0.2.0"
5
5
  end
data/qprocessor.gemspec CHANGED
@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
29
29
  spec.require_paths = ["lib"]
30
30
 
31
31
  # Uncomment to register a new dependency of your gem
32
- # spec.add_dependency "example-gem", "~> 1.0"
32
+ spec.add_dependency "aws-sdk-sqs", "~> 1.49"
33
+ spec.add_dependency "aws-sdk-sts", "~> 1.5"
33
34
  spec.add_dependency "beaneater", "~> 1.1"
34
35
 
35
36
  # For more information and examples about making a new gem, checkout our
metadata CHANGED
@@ -1,15 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qprocessor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Peter Wood
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-01-11 00:00:00.000000000 Z
11
+ date: 2022-01-14 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-sqs
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.49'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.49'
27
+ - !ruby/object:Gem::Dependency
28
+ name: aws-sdk-sts
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.5'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.5'
13
41
  - !ruby/object:Gem::Dependency
14
42
  name: beaneater
15
43
  requirement: !ruby/object:Gem::Requirement
@@ -42,8 +70,14 @@ files:
42
70
  - bin/console
43
71
  - bin/setup
44
72
  - lib/qprocessor.rb
73
+ - lib/qprocessor/beanstalk_message.rb
74
+ - lib/qprocessor/beanstalk_source.rb
45
75
  - lib/qprocessor/json_parser.rb
76
+ - lib/qprocessor/message.rb
46
77
  - lib/qprocessor/processor.rb
78
+ - lib/qprocessor/sources.rb
79
+ - lib/qprocessor/sqs_message.rb
80
+ - lib/qprocessor/sqs_source.rb
47
81
  - lib/qprocessor/version.rb
48
82
  - lib/qprocessor/yaml_parser.rb
49
83
  - qprocessor.gemspec