qprocessor 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +19 -1
- data/lib/qprocessor/beanstalk_message.rb +24 -0
- data/lib/qprocessor/beanstalk_source.rb +46 -0
- data/lib/qprocessor/message.rb +37 -0
- data/lib/qprocessor/processor.rb +12 -29
- data/lib/qprocessor/sources.rb +22 -0
- data/lib/qprocessor/sqs_message.rb +21 -0
- data/lib/qprocessor/sqs_source.rb +99 -0
- data/lib/qprocessor/version.rb +1 -1
- data/qprocessor.gemspec +2 -1
- metadata +36 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: acc989a7981c73a9503e369c54b4c912f3dab1ca073c33d49a38c379db2c0b7b
|
4
|
+
data.tar.gz: c408d6c67f41b2e0da1992aa8226d3b1bdd3e24c6ac9eda8bb91ed4010061af3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62068b04ecf60602b1094106c9fc0700659af07aa779b8f4b89e608a81f95e60fd29ef93257eee67f2cd0e1fa271bf25717017beed998b3b133c45a86f2e3d5d
|
7
|
+
data.tar.gz: ddeb4a8105387ec43a35a07edae349f6952a71e9b15d64b6e03b6bf0414bccfa57523067984610c36ef8422ff745358ac56ccd20e3c77c469e5006c30df6ac44
|
data/Gemfile.lock
CHANGED
@@ -1,13 +1,31 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
qprocessor (0.
|
4
|
+
qprocessor (0.2.0)
|
5
|
+
aws-sdk-sqs (~> 1.49)
|
6
|
+
aws-sdk-sts (~> 1.5)
|
5
7
|
beaneater (~> 1.1)
|
6
8
|
|
7
9
|
GEM
|
8
10
|
remote: https://rubygems.org/
|
9
11
|
specs:
|
12
|
+
aws-eventstream (1.2.0)
|
13
|
+
aws-partitions (1.548.0)
|
14
|
+
aws-sdk-core (3.125.3)
|
15
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
16
|
+
aws-partitions (~> 1, >= 1.525.0)
|
17
|
+
aws-sigv4 (~> 1.1)
|
18
|
+
jmespath (~> 1.0)
|
19
|
+
aws-sdk-sqs (1.49.0)
|
20
|
+
aws-sdk-core (~> 3, >= 3.125.0)
|
21
|
+
aws-sigv4 (~> 1.1)
|
22
|
+
aws-sdk-sts (1.5.0)
|
23
|
+
aws-sdk-core (~> 3, >= 3.110.0)
|
24
|
+
aws-sigv4 (~> 1.1)
|
25
|
+
aws-sigv4 (1.4.0)
|
26
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
10
27
|
beaneater (1.1.1)
|
28
|
+
jmespath (1.5.0)
|
11
29
|
rake (13.0.6)
|
12
30
|
|
13
31
|
PLATFORMS
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require "qprocessor/message"
|
2
|
+
|
3
|
+
module QProcessor
|
4
|
+
class BeanstalkMessage < Message
|
5
|
+
def initialize(source)
|
6
|
+
super(source)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Disposes of the underlying job, deleting it from Beanstalk.
|
10
|
+
def dispose
|
11
|
+
source.delete
|
12
|
+
end
|
13
|
+
|
14
|
+
# Retrieve the message identifier.
|
15
|
+
def id
|
16
|
+
source.id
|
17
|
+
end
|
18
|
+
|
19
|
+
# Releases the job back to Beanstalk.
|
20
|
+
def release
|
21
|
+
source.release
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "beaneater"
|
2
|
+
require "qprocessor/beanstalk_message"
|
3
|
+
|
4
|
+
module QProcessor
|
5
|
+
class BeanstalkSource
|
6
|
+
# A constant for a regular expression to match Beanstalk URLs
|
7
|
+
BEANSTALK_URL_PATTERN = /^beanstalk:\/\/([^:\/]+)[:]?([\d]*)[\/]?(.*)$/
|
8
|
+
|
9
|
+
# The default Beanstalk tube name.
|
10
|
+
DEFAULT_TUBE_NAME = "default"
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@name = "Beanstalk Source"
|
14
|
+
end
|
15
|
+
attr_reader :name
|
16
|
+
|
17
|
+
# This method blocks until a job is available, at which points it is
|
18
|
+
# returned.
|
19
|
+
def get
|
20
|
+
job = connection.reserve
|
21
|
+
message = QProcessor::BeanstalkMessage.new(job)
|
22
|
+
yield message if block_given?
|
23
|
+
message
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def beanstalk_url
|
29
|
+
@beanstalk_url ||= ENV["BEANSTALK_URL"]
|
30
|
+
end
|
31
|
+
|
32
|
+
def connection
|
33
|
+
if !@connection
|
34
|
+
match = BEANSTALK_URL_PATTERN.match(beanstalk_url)
|
35
|
+
raise QProcessor::Error("Invalid Beanstalk queue URL '#{beanstalk_url}' specified.") if !match
|
36
|
+
beanstalk = Beaneater.new("#{match[1]}#{match[2] == "" ? "" : ":#{match[2]}"}")
|
37
|
+
@connection = beanstalk.tubes[match[3] != "" ? match[3] : DEFAULT_TUBE_NAME]
|
38
|
+
end
|
39
|
+
@connection
|
40
|
+
end
|
41
|
+
|
42
|
+
def tube_name
|
43
|
+
ENV["BEANSTALK_TUBE_NAME"] || DEFAULT_TUBE_NAME
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module QProcessor
|
2
|
+
# This is a class representing a message or job pulled from a queue with the
|
3
|
+
# intention of this class providing a unified interface around different
|
4
|
+
# library classes.
|
5
|
+
class Message
|
6
|
+
def initialize(source)
|
7
|
+
@source = source
|
8
|
+
end
|
9
|
+
attr_reader :source
|
10
|
+
|
11
|
+
# Retrieves the content of the message/job as a String.
|
12
|
+
def body
|
13
|
+
@source.body
|
14
|
+
end
|
15
|
+
|
16
|
+
# This method should do whatever is necessary to clean up a message/job such
|
17
|
+
# that the underlying queuing mechanism will no longer deliver it. This is
|
18
|
+
# a 'do nothing' implementation that can be overridden by derived classes to
|
19
|
+
# provide for this functionality.
|
20
|
+
def dispose
|
21
|
+
end
|
22
|
+
|
23
|
+
# Fetches a unique identifer for the message/job. This default implementation
|
24
|
+
# raises an exception. Derived classes can override this to provide the
|
25
|
+
# required functionality.
|
26
|
+
def id
|
27
|
+
raise "The #{self.class.name} class does not override the #id() method."
|
28
|
+
end
|
29
|
+
|
30
|
+
# This method should do whatever is necessary to return a message/job to the
|
31
|
+
# underlying queuing mechanism such that it becomes available again for
|
32
|
+
# delivery. This is a 'do nothing' implementation that can be overridden by
|
33
|
+
# derived classes to provide for this functionality.
|
34
|
+
def release
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/qprocessor/processor.rb
CHANGED
@@ -1,27 +1,21 @@
|
|
1
1
|
require "beaneater"
|
2
2
|
require "logger"
|
3
|
+
require "qprocessor/sources"
|
3
4
|
|
4
5
|
module QProcessor
|
5
6
|
# This class encapsulates a lot of the functionality around creating a queue
|
6
7
|
# processor that pulls work from a queue and passes it to an instance of a
|
7
8
|
# specified class for 'handling'.
|
8
9
|
class Processor
|
9
|
-
# A constant for a regular expression to match Beanstalk URLs
|
10
|
-
BEANSTALK_URL_PATTERN = /^beanstalk:\/\/([^:\/]+)[:]?([\d]*)[\/]?(.*)$/
|
11
|
-
|
12
10
|
# A constant with the name of the default job parser class.
|
13
11
|
DEFAULT_PARSER_CLASS = "QProcessor::YAMLParser"
|
14
12
|
|
15
|
-
# The default Beanstalk tube name.
|
16
|
-
DEFAULT_TUBE_NAME = "default"
|
17
|
-
|
18
13
|
# A constant containing the maximum permitted setting for the inter-error sleep interval.
|
19
14
|
MAX_ERROR_RESTART_INTERVAL = 32
|
20
15
|
|
21
16
|
# Constructor for the Processor class.
|
22
17
|
#
|
23
18
|
# @param [String] The name assigned to the processor, primarily used for logging.
|
24
|
-
# @param [String] The URL to be used when connecting to the queue system.
|
25
19
|
# @param [Hash] A Hash of the class names used by the processor. There are two
|
26
20
|
# keys recognised in this Hash - :parser and :processor. The
|
27
21
|
# :parser class should be the fully qualified name of the class
|
@@ -30,12 +24,11 @@ module QProcessor
|
|
30
24
|
# process jobs.
|
31
25
|
# @param [Hash] A Hash of additional settings that will be used by the processor.
|
32
26
|
# Valid keys include :logger and :reuse_processor.
|
33
|
-
def initialize(name,
|
27
|
+
def initialize(name, class_names, settings={})
|
34
28
|
@instance = nil
|
35
29
|
@name = name
|
36
30
|
@parser_class = get_class!(class_names.fetch(:parser, DEFAULT_PARSER_CLASS))
|
37
31
|
@processor_class = get_class!(class_names[:processor])
|
38
|
-
@queue_url = url
|
39
32
|
@settings = {}.merge(settings)
|
40
33
|
@terminate = false
|
41
34
|
end
|
@@ -44,15 +37,20 @@ module QProcessor
|
|
44
37
|
# Starts processing of the queue. This method will not return to the caller
|
45
38
|
# as it will run a loop processing the queue jobs as they become available.
|
46
39
|
def start
|
47
|
-
queue =
|
40
|
+
queue = QProcessor::Sources.manufacture!
|
48
41
|
interval = 0
|
49
42
|
while !@terminate
|
50
43
|
begin
|
51
44
|
logger.debug("The '#{name}' queue processor is listening for jobs.")
|
52
|
-
queue.
|
53
|
-
|
54
|
-
|
55
|
-
|
45
|
+
queue.get do|message|
|
46
|
+
begin
|
47
|
+
logger.debug "The '#{name}' queue processor received message id #{message.id}."
|
48
|
+
handle(message)
|
49
|
+
interval = 0
|
50
|
+
rescue => error
|
51
|
+
message.release
|
52
|
+
raise
|
53
|
+
end
|
56
54
|
end
|
57
55
|
rescue => error
|
58
56
|
logger.error "The '#{name}' queue processor caught an exception.\nType: #{error.class.name}\n"\
|
@@ -66,21 +64,6 @@ module QProcessor
|
|
66
64
|
|
67
65
|
private
|
68
66
|
|
69
|
-
def beanstalk_connect
|
70
|
-
match = BEANSTALK_URL_PATTERN.match(queue_url)
|
71
|
-
raise QProcessor::Error("Invalid Beanstalk queue URL '#{queue_url}' specified.") if !match
|
72
|
-
beanstalk = Beaneater.new("#{match[1]}#{match[2] == "" ? "" : ":#{match[2]}"}")
|
73
|
-
beanstalk.tubes[match[3] != "" ? match[3] : DEFAULT_TUBE_NAME]
|
74
|
-
end
|
75
|
-
|
76
|
-
def connect
|
77
|
-
if queue_url[0, 9] == "beanstalk"
|
78
|
-
beanstalk_connect
|
79
|
-
else
|
80
|
-
raise QProcessor::Error.new("Unrecognised queue URL '#{queue_url}' encountered")
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
67
|
def get_class(name)
|
85
68
|
name.split("::").reduce(Object) {|t, e| (!t.nil? ? t.const_get(e) : nil)}
|
86
69
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require "qprocessor/beanstalk_source"
|
2
|
+
require "qprocessor/sqs_source"
|
3
|
+
|
4
|
+
module QProcessor
|
5
|
+
module Sources
|
6
|
+
# This method checks environment variables to see if any are set such that
|
7
|
+
# a queue source can be generated from them. When checking environment
|
8
|
+
# variables Beanstalk has highest priority, followed by SQS. If no matching
|
9
|
+
# environment variables are found then an exception will be raised. For
|
10
|
+
# Beanstalk the BEANSTALK_URL environment variable is checked for. For SQS
|
11
|
+
# the SQS_QUEUE_NAME environment variable is check for.
|
12
|
+
def self.manufacture!
|
13
|
+
if !ENV["BEANSTALK_URL"].nil?
|
14
|
+
QProcessor::BeanstalkSource.new
|
15
|
+
elsif !ENV["SQS_QUEUE_NAME"].nil?
|
16
|
+
QProcessor::SQSSource.new
|
17
|
+
else
|
18
|
+
raise "Unable to determine source queue type from environment settings."
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require "qprocessor/message"
|
2
|
+
|
3
|
+
module QProcessor
|
4
|
+
class SQSMessage < Message
|
5
|
+
def initialize(source, client, queue_url)
|
6
|
+
super(source)
|
7
|
+
@client = client
|
8
|
+
@queue_url = queue_url
|
9
|
+
end
|
10
|
+
|
11
|
+
# Deletes a message from the source queue.
|
12
|
+
def dispose
|
13
|
+
@client.delete_message(queue_url: @queue_url, receipt_handle: source.receipt_handle)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Retrieves the message identifier.
|
17
|
+
def id
|
18
|
+
source.message_id
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require "aws-sdk-sqs"
|
2
|
+
require "aws-sdk-sts"
|
3
|
+
require "qprocessor/sqs_message"
|
4
|
+
require "pp"
|
5
|
+
|
6
|
+
module QProcessor
|
7
|
+
class SQSSource
|
8
|
+
# A constant for the maximum wait for message timeout to be used by the
|
9
|
+
# source.
|
10
|
+
MAX_MESSAGE_WAIT = 3
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@name = "SQS Source"
|
14
|
+
end
|
15
|
+
attr_reader :name
|
16
|
+
|
17
|
+
# This method blocks until a job is available, at which points it is
|
18
|
+
# returned.
|
19
|
+
def get
|
20
|
+
message = nil
|
21
|
+
client = sqs_client
|
22
|
+
url = sqs_queue_url(sqs_queue_name)
|
23
|
+
while message.nil?
|
24
|
+
result = client.receive_message(max_number_of_messages: 1,
|
25
|
+
queue_url: url,
|
26
|
+
wait_time_seconds: MAX_MESSAGE_WAIT)
|
27
|
+
message = QProcessor::SQSMessage.new(result.messages[0], client, url) if !result.messages.empty?
|
28
|
+
end
|
29
|
+
yield message if block_given?
|
30
|
+
message
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
# Attempts to fetch the AWS access key from environment variables, raising
|
36
|
+
# an exception if not found.
|
37
|
+
def aws_access_key
|
38
|
+
@aws_access_key ||= ENV["AWS_ACCESS_KEY"]
|
39
|
+
raise "The AWS_ACCESS_KEY environment variable has not been set." if @aws_access_key.nil?
|
40
|
+
@aws_access_key
|
41
|
+
end
|
42
|
+
|
43
|
+
# Creates a Aws::Credentials instance from AWS settings read from the
|
44
|
+
# environment. An exception will be raised if any of these settings is
|
45
|
+
# not present.
|
46
|
+
def aws_credentials
|
47
|
+
Aws::Credentials.new(aws_access_key, aws_secret_key)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Attempts to fetch the AWS region from environment variables, raising an
|
51
|
+
# exception if not found.
|
52
|
+
def aws_region
|
53
|
+
@aws_region ||= ENV["AWS_REGION"]
|
54
|
+
raise "The AWS_REGION environment variable has not been set." if @aws_region.nil?
|
55
|
+
@aws_region
|
56
|
+
end
|
57
|
+
|
58
|
+
# Attempts to fetch the AWS secret key from environment variables, raising
|
59
|
+
# an exception if not found.
|
60
|
+
def aws_secret_key
|
61
|
+
@aws_secret_key ||= ENV["AWS_SECRET_KEY"]
|
62
|
+
raise "The AWS_SECRET_KEY environment variable has not been set." if @aws_secret_key.nil?
|
63
|
+
@aws_secret_key
|
64
|
+
end
|
65
|
+
|
66
|
+
def connection
|
67
|
+
if !@connection
|
68
|
+
url = beanstalk_url
|
69
|
+
raise "No BEANSTALK_URL environment variable set." if url.nil?
|
70
|
+
@connection ||= Beaneater.new(url)
|
71
|
+
end
|
72
|
+
@connection
|
73
|
+
end
|
74
|
+
|
75
|
+
# Fetches a Aws::SQS::Client instance using AWS credentials and region
|
76
|
+
# details that are read from environment settings. If any of these settings
|
77
|
+
# are missing an exception will be raised.
|
78
|
+
def sqs_client
|
79
|
+
Aws::SQS::Client.new(credentials: aws_credentials, region: aws_region)
|
80
|
+
end
|
81
|
+
|
82
|
+
# Attempts to fetch the SQS queue name from environment variables, raising
|
83
|
+
# an exception if not found.
|
84
|
+
def sqs_queue_name
|
85
|
+
@sqs_queue_name ||= ENV["SQS_QUEUE_NAME"]
|
86
|
+
raise "The SQS_QUEUE_NAME environment variable has not been set." if @sqs_queue_name.nil?
|
87
|
+
@sqs_queue_name
|
88
|
+
end
|
89
|
+
|
90
|
+
# Generates an URL for use with a specific AWS SQS queue.
|
91
|
+
def sqs_queue_url(queue_name)
|
92
|
+
if !@sqs_queue_url
|
93
|
+
client = Aws::STS::Client.new(credentials: aws_credentials, region: aws_region)
|
94
|
+
@sqs_queue_url = "https://sqs.#{aws_region}.amazonaws.com/#{client.get_caller_identity.account}/#{queue_name}"
|
95
|
+
end
|
96
|
+
@sqs_queue_url
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/qprocessor/version.rb
CHANGED
data/qprocessor.gemspec
CHANGED
@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.require_paths = ["lib"]
|
30
30
|
|
31
31
|
# Uncomment to register a new dependency of your gem
|
32
|
-
|
32
|
+
spec.add_dependency "aws-sdk-sqs", "~> 1.49"
|
33
|
+
spec.add_dependency "aws-sdk-sts", "~> 1.5"
|
33
34
|
spec.add_dependency "beaneater", "~> 1.1"
|
34
35
|
|
35
36
|
# For more information and examples about making a new gem, checkout our
|
metadata
CHANGED
@@ -1,15 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qprocessor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Wood
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-01-
|
11
|
+
date: 2022-01-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-sqs
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.49'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.49'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: aws-sdk-sts
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.5'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.5'
|
13
41
|
- !ruby/object:Gem::Dependency
|
14
42
|
name: beaneater
|
15
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -42,8 +70,14 @@ files:
|
|
42
70
|
- bin/console
|
43
71
|
- bin/setup
|
44
72
|
- lib/qprocessor.rb
|
73
|
+
- lib/qprocessor/beanstalk_message.rb
|
74
|
+
- lib/qprocessor/beanstalk_source.rb
|
45
75
|
- lib/qprocessor/json_parser.rb
|
76
|
+
- lib/qprocessor/message.rb
|
46
77
|
- lib/qprocessor/processor.rb
|
78
|
+
- lib/qprocessor/sources.rb
|
79
|
+
- lib/qprocessor/sqs_message.rb
|
80
|
+
- lib/qprocessor/sqs_source.rb
|
47
81
|
- lib/qprocessor/version.rb
|
48
82
|
- lib/qprocessor/yaml_parser.rb
|
49
83
|
- qprocessor.gemspec
|