rservicebus2 0.2.8 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rservicebus2.rb +6 -5
- data/lib/rservicebus2/agent.rb +7 -4
- data/lib/rservicebus2/appresource.rb +1 -1
- data/lib/rservicebus2/appresource/awss3.rb +3 -3
- data/lib/rservicebus2/appresource/awssqs.rb +2 -0
- data/lib/rservicebus2/appresource/dir.rb +3 -0
- data/lib/rservicebus2/appresource/file.rb +2 -0
- data/lib/rservicebus2/appresource/fluiddb.rb +3 -0
- data/lib/rservicebus2/appresource/fluiddb2.rb +3 -1
- data/lib/rservicebus2/appresource_configure.rb +8 -4
- data/lib/rservicebus2/audit.rb +4 -2
- data/lib/rservicebus2/config.rb +39 -27
- data/lib/rservicebus2/endpointmapping.rb +2 -4
- data/lib/rservicebus2/errormessage.rb +1 -1
- data/lib/rservicebus2/handler_manager.rb +1 -1
- data/lib/rservicebus2/helper_functions.rb +19 -14
- data/lib/rservicebus2/host.rb +62 -123
- data/lib/rservicebus2/message.rb +11 -19
- data/lib/rservicebus2/message/statisticoutput.rb +2 -0
- data/lib/rservicebus2/message/subscription.rb +2 -0
- data/lib/rservicebus2/message/verboseoutput.rb +2 -0
- data/lib/rservicebus2/monitor.rb +7 -7
- data/lib/rservicebus2/monitor/awss3.rb +3 -1
- data/lib/rservicebus2/monitor/awssqs.rb +9 -7
- data/lib/rservicebus2/monitor/dir.rb +25 -23
- data/lib/rservicebus2/monitor/dirnotifier.rb +4 -1
- data/lib/rservicebus2/monitor/message.rb +2 -0
- data/lib/rservicebus2/monitor/xmldir.rb +2 -0
- data/lib/rservicebus2/monitor_configure.rb +46 -36
- data/lib/rservicebus2/mq.rb +27 -30
- data/lib/rservicebus2/mq/beanstalk.rb +5 -1
- data/lib/rservicebus2/saga/data.rb +1 -1
- data/lib/rservicebus2/saga/manager.rb +1 -1
- data/lib/rservicebus2/saga_loader.rb +27 -24
- data/lib/rservicebus2/saga_storage.rb +5 -4
- data/lib/rservicebus2/sendat_manager.rb +5 -3
- data/lib/rservicebus2/sendat_storage.rb +5 -3
- data/lib/rservicebus2/sendat_storage/file.rb +6 -2
- data/lib/rservicebus2/sendat_storage/inmemory.rb +2 -0
- data/lib/rservicebus2/state_manager.rb +5 -4
- data/lib/rservicebus2/state_storage.rb +7 -6
- data/lib/rservicebus2/statistic_manager.rb +7 -3
- data/lib/rservicebus2/subscription_manager.rb +7 -5
- data/lib/rservicebus2/subscription_storage.rb +6 -5
- data/lib/rservicebus2/subscription_storage/file.rb +5 -18
- data/lib/rservicebus2/subscription_storage_configure.rb +2 -0
- data/lib/rservicebus2/transporter.rb +63 -52
- metadata +2 -3
- data/lib/rservicebus2/stats.rb +0 -68
data/lib/rservicebus2/monitor.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RServiceBus2
|
2
4
|
# Monitor
|
3
5
|
class Monitor
|
@@ -5,12 +7,12 @@ module RServiceBus2
|
|
5
7
|
|
6
8
|
# The method which actually connects to the resource.
|
7
9
|
def connect(_uri)
|
8
|
-
|
10
|
+
raise 'Method, connect, needs to be implemented for resource'
|
9
11
|
end
|
10
12
|
|
11
13
|
# The method which actually connects to the resource.
|
12
14
|
def look
|
13
|
-
|
15
|
+
raise 'Method, Look, needs to be implemented for the Monitor'
|
14
16
|
end
|
15
17
|
|
16
18
|
def _connect
|
@@ -34,8 +36,7 @@ module RServiceBus2
|
|
34
36
|
end
|
35
37
|
|
36
38
|
# A notification that allows cleanup
|
37
|
-
def finished
|
38
|
-
end
|
39
|
+
def finished; end
|
39
40
|
|
40
41
|
# At least called in the Host rescue block, to ensure all network links
|
41
42
|
# are healthy
|
@@ -43,9 +44,8 @@ module RServiceBus2
|
|
43
44
|
begin
|
44
45
|
finished
|
45
46
|
rescue StandardError => e
|
46
|
-
puts
|
47
|
-
|
48
|
-
puts 'Message: ' + e.message
|
47
|
+
puts "** Monitor. An error was raised while closing connection to, #{@uri}"
|
48
|
+
puts "Message: #{e.message}"
|
49
49
|
puts e.backtrace
|
50
50
|
end
|
51
51
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sdk-s3'
|
2
4
|
|
3
5
|
module RServiceBus2
|
@@ -36,7 +38,7 @@ module RServiceBus2
|
|
36
38
|
file_processed = 0
|
37
39
|
max_files_processed = 2
|
38
40
|
|
39
|
-
objects = @s3_client.list_objects_v2(
|
41
|
+
objects = @s3_client.list_objects_v2(bucket: @bucket_name, max_keys: max_files_processed).contents
|
40
42
|
|
41
43
|
objects.each do |object|
|
42
44
|
RServiceBus2.log "Ready to process, #{object.key}"
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sdk-sqs'
|
2
4
|
require 'aws-sdk-sts'
|
3
5
|
|
@@ -12,20 +14,20 @@ module RServiceBus2
|
|
12
14
|
sts_client = Aws::STS::Client.new(region: region)
|
13
15
|
caller_identity_account = sts_client.get_caller_identity.account
|
14
16
|
|
15
|
-
@queue_url = "https://sqs.#{region}.amazonaws.com
|
16
|
-
"#{caller_identity_account}/#{queue_name}"
|
17
|
+
@queue_url = "https://sqs.#{region}.amazonaws.com/#{caller_identity_account}/#{queue_name}"
|
17
18
|
@sqs_client = Aws::SQS::Client.new(region: region)
|
18
19
|
end
|
19
20
|
|
20
21
|
def look
|
21
|
-
# TODO make max available as env variable
|
22
22
|
response = @sqs_client.receive_message(queue_url: @queue_url, max_number_of_messages: 1)
|
23
23
|
response.messages.each do |message|
|
24
24
|
send(message.body, URI.parse(CGI.escape(@queue_url)))
|
25
|
-
@sqs_client.delete_message(
|
26
|
-
|
27
|
-
|
28
|
-
|
25
|
+
@sqs_client.delete_message(
|
26
|
+
{
|
27
|
+
queue_url: @queue_url,
|
28
|
+
receipt_handle: message.receipt_handle
|
29
|
+
}
|
30
|
+
)
|
29
31
|
end
|
30
32
|
end
|
31
33
|
end
|
@@ -1,38 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'cgi'
|
2
4
|
require 'zip/zip'
|
3
5
|
require 'zlib'
|
4
6
|
|
5
7
|
module RServiceBus2
|
6
8
|
# Monitor Directory for files
|
9
|
+
|
7
10
|
# rubocop:disable Metrics/ClassLength
|
8
11
|
class MonitorDir < Monitor
|
9
|
-
def
|
12
|
+
def input_dir(uri)
|
10
13
|
# Pass the path through the Dir object to check syntax on startup
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
and create it as a directory."
|
28
|
-
puts "***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
29
|
-
abort
|
30
|
-
end
|
14
|
+
return Dir.new(uri.path) if File.writable?(uri.path)
|
15
|
+
|
16
|
+
puts "***** Directory is not writable, #{uri.path}.\n" \
|
17
|
+
"***** Make the directory, #{uri.path}, writable and try again."
|
18
|
+
abort
|
19
|
+
rescue Errno::ENOENT
|
20
|
+
puts "***** Directory does not exist, #{uri.path}.\n" \
|
21
|
+
"***** Create the directory, #{uri.path}, and try again.\n" \
|
22
|
+
"***** eg, mkdir #{uri.path}"
|
23
|
+
abort
|
24
|
+
rescue Errno::ENOTDIR
|
25
|
+
puts "***** The specified path does not point to a directory, #{uri.path}.\n" \
|
26
|
+
"***** Either repoint path to a directory, or remove, #{uri.path}, and create it as a directory.\n" \
|
27
|
+
"***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
28
|
+
abort
|
29
|
+
end
|
31
30
|
|
32
|
-
|
31
|
+
def connect(uri)
|
32
|
+
@path = input_dir(uri).path
|
33
33
|
@input_filter = []
|
34
34
|
|
35
35
|
return if uri.query.nil?
|
36
|
+
|
36
37
|
parts = CGI.parse(uri.query)
|
37
38
|
@querystringparts = parts
|
38
39
|
if parts.key?('archive')
|
@@ -124,7 +125,7 @@ module RServiceBus2
|
|
124
125
|
|
125
126
|
unless @archivedir.nil?
|
126
127
|
basename = File.basename(file_path)
|
127
|
-
new_file_path = "#{@archivedir}/#{basename}.#{
|
128
|
+
new_file_path = "#{@archivedir}/#{basename}.#{Time.now.strftime('%Y%m%d%H%M%S%L')}.zip"
|
128
129
|
RServiceBus2.log "Writing to archive, #{new_file_path}"
|
129
130
|
|
130
131
|
Zip::ZipOutputStream.open(new_file_path) do |zos|
|
@@ -141,4 +142,5 @@ module RServiceBus2
|
|
141
142
|
end
|
142
143
|
end
|
143
144
|
end
|
145
|
+
# rubocop:enable Metrics/ClassLength
|
144
146
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'cgi'
|
2
4
|
require 'fileutils'
|
3
5
|
require 'pathname'
|
@@ -6,6 +8,7 @@ module RServiceBus2
|
|
6
8
|
# Monitor for Directory
|
7
9
|
class MonitorDirNotifier < Monitor
|
8
10
|
attr_reader :path, :processing_folder, :filter
|
11
|
+
|
9
12
|
def connect(uri)
|
10
13
|
# Pass the path through the Dir object to check syntax on startup
|
11
14
|
begin
|
@@ -45,7 +48,7 @@ module RServiceBus2
|
|
45
48
|
begin
|
46
49
|
open_folder processing_uri.path
|
47
50
|
unless File.writable?(processing_uri.path)
|
48
|
-
puts "*****
|
51
|
+
puts "***** 1Processing Directory is not writable,
|
49
52
|
#{processing_uri.path}."
|
50
53
|
puts "***** Make the directory, #{processing_uri.path},
|
51
54
|
writable and try again."
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'rservicebus2/monitor'
|
2
4
|
require 'rservicebus2/monitor/message'
|
3
5
|
|
@@ -16,56 +18,64 @@ module RServiceBus2
|
|
16
18
|
@resource_list = {}
|
17
19
|
end
|
18
20
|
|
21
|
+
def conditionally_set_app_resource(monitor, key, val)
|
22
|
+
monitor.instance_variable_set("@#{key}", val.resource)
|
23
|
+
@resource_list[monitor.class.name] = [] if @resource_list[monitor.class.name].nil?
|
24
|
+
@resource_list[monitor.class.name] << val
|
25
|
+
@host.log "App resource attribute, #{key}, set for: #{monitor.class.name}"
|
26
|
+
end
|
27
|
+
|
19
28
|
# Assigns appropriate resources to writable attributes in the handler that
|
20
29
|
# match keys in the resource hash
|
21
30
|
# @param [RServiceBus2::Handler] handler
|
22
|
-
def
|
31
|
+
def conditionally_set_app_resources(monitor)
|
23
32
|
RServiceBus2.rlog "Checking app resources for: #{monitor.class.name}"
|
24
|
-
RServiceBus2.rlog "If your attribute is not getting set, check that it is
|
25
|
-
in the 'attr_accessor' list"
|
33
|
+
RServiceBus2.rlog "If your attribute is not getting set, check that it is in the 'attr_accessor' list"
|
26
34
|
@resource_manager.all.each do |k, v|
|
27
35
|
next unless monitor.class.method_defined?(k)
|
28
36
|
|
29
|
-
monitor
|
30
|
-
@resource_list[monitor.class.name] = [] if
|
31
|
-
@resource_list[monitor.class.name].nil?
|
32
|
-
@resource_list[monitor.class.name] << v
|
33
|
-
@host.log "App resource attribute, #{k}, set for: " +
|
34
|
-
monitor.class.name
|
37
|
+
conditionally_set_app_resource(monitor, k, v)
|
35
38
|
end
|
36
39
|
self
|
37
40
|
end
|
38
41
|
|
39
|
-
|
40
|
-
|
42
|
+
# rubocop:disable Metrics/MethodLength
|
43
|
+
def load_monitor(key, val)
|
44
|
+
name = key.sub('RSBOB_', '')
|
45
|
+
uri = URI.parse(val)
|
46
|
+
monitor = nil
|
47
|
+
case uri.scheme
|
48
|
+
when 'dir'
|
49
|
+
require 'rservicebus2/monitor/dir'
|
50
|
+
monitor = MonitorDir.new(@host, name, uri)
|
51
|
+
when 'awss3'
|
52
|
+
require 'rservicebus2/monitor/awss3'
|
53
|
+
monitor = MonitorAWSS3.new(@host, name, uri)
|
54
|
+
when 'awssqs'
|
55
|
+
require 'rservicebus2/monitor/awssqs'
|
56
|
+
monitor = MonitorAWSSQS.new(@host, name, uri)
|
57
|
+
when 'dirnotifier'
|
58
|
+
require 'rservicebus2/monitor/dirnotifier'
|
59
|
+
monitor = MonitorDirNotifier.new(@host, name, uri)
|
60
|
+
else
|
61
|
+
abort("Scheme, #{uri.scheme}, not recognised when configuring
|
62
|
+
Monitor, #{key}=#{val}")
|
63
|
+
end
|
64
|
+
monitor
|
65
|
+
end
|
66
|
+
# rubocop:enable Metrics/MethodLength
|
67
|
+
|
68
|
+
def monitors(env)
|
69
|
+
list = []
|
41
70
|
|
42
71
|
env.each do |k, v|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
when 'dir'
|
49
|
-
require 'rservicebus2/monitor/dir'
|
50
|
-
monitor = MonitorDir.new(@host, name, uri)
|
51
|
-
when 'awss3'
|
52
|
-
require 'rservicebus2/monitor/awss3'
|
53
|
-
monitor = MonitorAWSS3.new(@host, name, uri)
|
54
|
-
when 'awssqs'
|
55
|
-
require 'rservicebus2/monitor/awssqs'
|
56
|
-
monitor = MonitorAWSSQS.new(@host, name, uri)
|
57
|
-
when 'dirnotifier'
|
58
|
-
require 'rservicebus2/monitor/dirnotifier'
|
59
|
-
monitor = MonitorDirNotifier.new(@host, name, uri)
|
60
|
-
else
|
61
|
-
abort("Scheme, #{uri.scheme}, not recognised when configuring
|
62
|
-
Monitor, #{k}=#{v}")
|
63
|
-
end
|
64
|
-
set_app_resources(monitor)
|
65
|
-
monitors << monitor
|
66
|
-
end
|
72
|
+
next unless v.is_a?(String) && k.start_with?('RSBOB_')
|
73
|
+
|
74
|
+
monitor = load_monitor(k, v)
|
75
|
+
conditionally_set_app_resources(monitor)
|
76
|
+
list << monitor
|
67
77
|
end
|
68
|
-
|
78
|
+
list
|
69
79
|
end
|
70
80
|
end
|
71
81
|
end
|
data/lib/rservicebus2/mq.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
4
|
|
3
5
|
module RServiceBus2
|
@@ -10,6 +12,7 @@ module RServiceBus2
|
|
10
12
|
class MQ
|
11
13
|
attr_reader :local_queue_name
|
12
14
|
|
15
|
+
# rubocop:disable Metrics/MethodLength
|
13
16
|
def self.get
|
14
17
|
mq_string = RServiceBus2.get_value('RSBMQ', 'beanstalk://localhost')
|
15
18
|
uri = URI.parse(mq_string)
|
@@ -34,31 +37,26 @@ module RServiceBus2
|
|
34
37
|
|
35
38
|
mq
|
36
39
|
end
|
40
|
+
# rubocop:enable Metrics/MethodLength
|
37
41
|
|
38
42
|
# Resources are attached, and are be specified using the URI syntax
|
39
43
|
# @param [URI] uri the type and location of queue,
|
40
44
|
# eg beanstalk://127.0.0.1/foo
|
41
45
|
# @param [Integer] timeout the amount of time to wait for a msg to arrive
|
46
|
+
|
47
|
+
# rubocop:disable Metrics/MethodLength
|
42
48
|
def initialize(uri)
|
43
|
-
|
44
|
-
@uri = uri
|
45
|
-
else
|
46
|
-
puts 'uri must be a valid URI'
|
47
|
-
abort
|
48
|
-
end
|
49
|
+
abort 'Paramter to mq must be a valid URI' unless uri.is_a? URI
|
49
50
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
@local_queue_name[0] = ''
|
55
|
-
end
|
51
|
+
@uri = uri
|
52
|
+
@local_queue_name = uri.path
|
53
|
+
@local_queue_name[0] = '' if @local_queue_name[0] == '/'
|
54
|
+
@local_queue_name = RServiceBus2.get_value('APPNAME', 'RServiceBus') if @local_queue_name == ''
|
56
55
|
|
57
56
|
if @local_queue_name == ''
|
58
|
-
puts
|
59
|
-
|
60
|
-
|
61
|
-
puts '*** the structure is scheme://host[:port]/queuename'
|
57
|
+
puts 'Queue name must be supplied ' \
|
58
|
+
"*** uri, #{uri}, needs to contain a queue name" \
|
59
|
+
'*** the structure is scheme://host[:port]/queuename'
|
62
60
|
abort
|
63
61
|
end
|
64
62
|
|
@@ -66,42 +64,41 @@ module RServiceBus2
|
|
66
64
|
connect(uri.host, uri.port)
|
67
65
|
subscribe(@local_queue_name)
|
68
66
|
end
|
67
|
+
# rubocop:enable Metrics/MethodLength
|
69
68
|
|
70
69
|
# Connect to the broker
|
71
70
|
# @param [String] host machine runnig the mq
|
72
71
|
# @param [String] port port the mq is running on
|
73
72
|
def connect(_host, _port)
|
74
|
-
|
73
|
+
raise 'Method, connect, needs to be implemented'
|
75
74
|
end
|
76
75
|
|
77
76
|
# Connect to the receiving queue
|
78
77
|
# @param [String] queuename name of the receiving queue
|
79
78
|
def subscribe(_queuename)
|
80
|
-
|
79
|
+
raise 'Method, subscribe, needs to be implemented'
|
81
80
|
end
|
82
81
|
|
83
82
|
# Get next msg from queue
|
84
83
|
def pop
|
85
|
-
|
84
|
+
raise 'Method, pop, needs to be implemented'
|
86
85
|
end
|
87
86
|
|
88
87
|
# "Commit" the pop
|
89
88
|
def ack
|
90
|
-
|
89
|
+
raise 'Method, ack, needs to be implemented'
|
91
90
|
end
|
92
91
|
|
93
92
|
# At least called in the Host rescue block, to ensure all network links are
|
94
93
|
# healthy
|
95
94
|
# @param [String] queue_name name of the queue to which the msg should be sent
|
96
95
|
# @param [String] msg msg to be sent
|
97
|
-
def send(
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
end
|
105
|
-
|
106
|
-
end
|
96
|
+
def send(_queue_name, _msg)
|
97
|
+
@connection.close
|
98
|
+
rescue StandardError => e
|
99
|
+
puts "AppResource. An error was raised while closing connection to, #{@uri}"
|
100
|
+
puts "Error: #{e.message}"
|
101
|
+
puts "Backtrace: #{e.backtrace}"
|
102
|
+
end
|
103
|
+
end
|
107
104
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'beanstalk-client'
|
2
4
|
require 'rservicebus2/mq'
|
3
5
|
|
@@ -34,6 +36,7 @@ module RServiceBus2
|
|
34
36
|
abort
|
35
37
|
end
|
36
38
|
end
|
39
|
+
# rubocop:enable Metrics/AbcSize,Metrics/MethodLength
|
37
40
|
|
38
41
|
def subscribe(queuename)
|
39
42
|
@beanstalk.watch(queuename)
|
@@ -45,6 +48,7 @@ module RServiceBus2
|
|
45
48
|
@job = @beanstalk.reserve @timeout
|
46
49
|
rescue StandardError => e
|
47
50
|
raise NoMsgToProcess if e.message == 'TIMED_OUT'
|
51
|
+
|
48
52
|
raise e
|
49
53
|
end
|
50
54
|
@job.body
|
@@ -63,7 +67,7 @@ module RServiceBus2
|
|
63
67
|
if msg.length > @max_job_size
|
64
68
|
puts '***Attempting to send a msg which will not fit on queue.'
|
65
69
|
puts "***Msg size, #{msg.length}, max msg size, #{@max_job_size}."
|
66
|
-
|
70
|
+
raise JobTooBigError, "Msg size, #{msg.length}, max msg size,
|
67
71
|
#{@max_job_size}"
|
68
72
|
end
|
69
73
|
@beanstalk.use(queue_name)
|