rservicebus2 0.2.8 → 0.2.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rservicebus2.rb +6 -5
- data/lib/rservicebus2/agent.rb +7 -4
- data/lib/rservicebus2/appresource.rb +1 -1
- data/lib/rservicebus2/appresource/awss3.rb +3 -3
- data/lib/rservicebus2/appresource/awssqs.rb +2 -0
- data/lib/rservicebus2/appresource/dir.rb +3 -0
- data/lib/rservicebus2/appresource/file.rb +2 -0
- data/lib/rservicebus2/appresource/fluiddb.rb +3 -0
- data/lib/rservicebus2/appresource/fluiddb2.rb +3 -1
- data/lib/rservicebus2/appresource_configure.rb +8 -4
- data/lib/rservicebus2/audit.rb +4 -2
- data/lib/rservicebus2/config.rb +39 -27
- data/lib/rservicebus2/endpointmapping.rb +2 -4
- data/lib/rservicebus2/errormessage.rb +1 -1
- data/lib/rservicebus2/handler_manager.rb +1 -1
- data/lib/rservicebus2/helper_functions.rb +19 -14
- data/lib/rservicebus2/host.rb +62 -123
- data/lib/rservicebus2/message.rb +11 -19
- data/lib/rservicebus2/message/statisticoutput.rb +2 -0
- data/lib/rservicebus2/message/subscription.rb +2 -0
- data/lib/rservicebus2/message/verboseoutput.rb +2 -0
- data/lib/rservicebus2/monitor.rb +7 -7
- data/lib/rservicebus2/monitor/awss3.rb +3 -1
- data/lib/rservicebus2/monitor/awssqs.rb +9 -7
- data/lib/rservicebus2/monitor/dir.rb +25 -23
- data/lib/rservicebus2/monitor/dirnotifier.rb +4 -1
- data/lib/rservicebus2/monitor/message.rb +2 -0
- data/lib/rservicebus2/monitor/xmldir.rb +2 -0
- data/lib/rservicebus2/monitor_configure.rb +46 -36
- data/lib/rservicebus2/mq.rb +27 -30
- data/lib/rservicebus2/mq/beanstalk.rb +5 -1
- data/lib/rservicebus2/saga/data.rb +1 -1
- data/lib/rservicebus2/saga/manager.rb +1 -1
- data/lib/rservicebus2/saga_loader.rb +27 -24
- data/lib/rservicebus2/saga_storage.rb +5 -4
- data/lib/rservicebus2/sendat_manager.rb +5 -3
- data/lib/rservicebus2/sendat_storage.rb +5 -3
- data/lib/rservicebus2/sendat_storage/file.rb +6 -2
- data/lib/rservicebus2/sendat_storage/inmemory.rb +2 -0
- data/lib/rservicebus2/state_manager.rb +5 -4
- data/lib/rservicebus2/state_storage.rb +7 -6
- data/lib/rservicebus2/statistic_manager.rb +7 -3
- data/lib/rservicebus2/subscription_manager.rb +7 -5
- data/lib/rservicebus2/subscription_storage.rb +6 -5
- data/lib/rservicebus2/subscription_storage/file.rb +5 -18
- data/lib/rservicebus2/subscription_storage_configure.rb +2 -0
- data/lib/rservicebus2/transporter.rb +63 -52
- metadata +2 -3
- data/lib/rservicebus2/stats.rb +0 -68
data/lib/rservicebus2/monitor.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RServiceBus2
|
2
4
|
# Monitor
|
3
5
|
class Monitor
|
@@ -5,12 +7,12 @@ module RServiceBus2
|
|
5
7
|
|
6
8
|
# The method which actually connects to the resource.
|
7
9
|
def connect(_uri)
|
8
|
-
|
10
|
+
raise 'Method, connect, needs to be implemented for resource'
|
9
11
|
end
|
10
12
|
|
11
13
|
# The method which actually connects to the resource.
|
12
14
|
def look
|
13
|
-
|
15
|
+
raise 'Method, Look, needs to be implemented for the Monitor'
|
14
16
|
end
|
15
17
|
|
16
18
|
def _connect
|
@@ -34,8 +36,7 @@ module RServiceBus2
|
|
34
36
|
end
|
35
37
|
|
36
38
|
# A notification that allows cleanup
|
37
|
-
def finished
|
38
|
-
end
|
39
|
+
def finished; end
|
39
40
|
|
40
41
|
# At least called in the Host rescue block, to ensure all network links
|
41
42
|
# are healthy
|
@@ -43,9 +44,8 @@ module RServiceBus2
|
|
43
44
|
begin
|
44
45
|
finished
|
45
46
|
rescue StandardError => e
|
46
|
-
puts
|
47
|
-
|
48
|
-
puts 'Message: ' + e.message
|
47
|
+
puts "** Monitor. An error was raised while closing connection to, #{@uri}"
|
48
|
+
puts "Message: #{e.message}"
|
49
49
|
puts e.backtrace
|
50
50
|
end
|
51
51
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sdk-s3'
|
2
4
|
|
3
5
|
module RServiceBus2
|
@@ -36,7 +38,7 @@ module RServiceBus2
|
|
36
38
|
file_processed = 0
|
37
39
|
max_files_processed = 2
|
38
40
|
|
39
|
-
objects = @s3_client.list_objects_v2(
|
41
|
+
objects = @s3_client.list_objects_v2(bucket: @bucket_name, max_keys: max_files_processed).contents
|
40
42
|
|
41
43
|
objects.each do |object|
|
42
44
|
RServiceBus2.log "Ready to process, #{object.key}"
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sdk-sqs'
|
2
4
|
require 'aws-sdk-sts'
|
3
5
|
|
@@ -12,20 +14,20 @@ module RServiceBus2
|
|
12
14
|
sts_client = Aws::STS::Client.new(region: region)
|
13
15
|
caller_identity_account = sts_client.get_caller_identity.account
|
14
16
|
|
15
|
-
@queue_url = "https://sqs.#{region}.amazonaws.com
|
16
|
-
"#{caller_identity_account}/#{queue_name}"
|
17
|
+
@queue_url = "https://sqs.#{region}.amazonaws.com/#{caller_identity_account}/#{queue_name}"
|
17
18
|
@sqs_client = Aws::SQS::Client.new(region: region)
|
18
19
|
end
|
19
20
|
|
20
21
|
def look
|
21
|
-
# TODO make max available as env variable
|
22
22
|
response = @sqs_client.receive_message(queue_url: @queue_url, max_number_of_messages: 1)
|
23
23
|
response.messages.each do |message|
|
24
24
|
send(message.body, URI.parse(CGI.escape(@queue_url)))
|
25
|
-
@sqs_client.delete_message(
|
26
|
-
|
27
|
-
|
28
|
-
|
25
|
+
@sqs_client.delete_message(
|
26
|
+
{
|
27
|
+
queue_url: @queue_url,
|
28
|
+
receipt_handle: message.receipt_handle
|
29
|
+
}
|
30
|
+
)
|
29
31
|
end
|
30
32
|
end
|
31
33
|
end
|
@@ -1,38 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'cgi'
|
2
4
|
require 'zip/zip'
|
3
5
|
require 'zlib'
|
4
6
|
|
5
7
|
module RServiceBus2
|
6
8
|
# Monitor Directory for files
|
9
|
+
|
7
10
|
# rubocop:disable Metrics/ClassLength
|
8
11
|
class MonitorDir < Monitor
|
9
|
-
def
|
12
|
+
def input_dir(uri)
|
10
13
|
# Pass the path through the Dir object to check syntax on startup
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
and create it as a directory."
|
28
|
-
puts "***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
29
|
-
abort
|
30
|
-
end
|
14
|
+
return Dir.new(uri.path) if File.writable?(uri.path)
|
15
|
+
|
16
|
+
puts "***** Directory is not writable, #{uri.path}.\n" \
|
17
|
+
"***** Make the directory, #{uri.path}, writable and try again."
|
18
|
+
abort
|
19
|
+
rescue Errno::ENOENT
|
20
|
+
puts "***** Directory does not exist, #{uri.path}.\n" \
|
21
|
+
"***** Create the directory, #{uri.path}, and try again.\n" \
|
22
|
+
"***** eg, mkdir #{uri.path}"
|
23
|
+
abort
|
24
|
+
rescue Errno::ENOTDIR
|
25
|
+
puts "***** The specified path does not point to a directory, #{uri.path}.\n" \
|
26
|
+
"***** Either repoint path to a directory, or remove, #{uri.path}, and create it as a directory.\n" \
|
27
|
+
"***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
28
|
+
abort
|
29
|
+
end
|
31
30
|
|
32
|
-
|
31
|
+
def connect(uri)
|
32
|
+
@path = input_dir(uri).path
|
33
33
|
@input_filter = []
|
34
34
|
|
35
35
|
return if uri.query.nil?
|
36
|
+
|
36
37
|
parts = CGI.parse(uri.query)
|
37
38
|
@querystringparts = parts
|
38
39
|
if parts.key?('archive')
|
@@ -124,7 +125,7 @@ module RServiceBus2
|
|
124
125
|
|
125
126
|
unless @archivedir.nil?
|
126
127
|
basename = File.basename(file_path)
|
127
|
-
new_file_path = "#{@archivedir}/#{basename}.#{
|
128
|
+
new_file_path = "#{@archivedir}/#{basename}.#{Time.now.strftime('%Y%m%d%H%M%S%L')}.zip"
|
128
129
|
RServiceBus2.log "Writing to archive, #{new_file_path}"
|
129
130
|
|
130
131
|
Zip::ZipOutputStream.open(new_file_path) do |zos|
|
@@ -141,4 +142,5 @@ module RServiceBus2
|
|
141
142
|
end
|
142
143
|
end
|
143
144
|
end
|
145
|
+
# rubocop:enable Metrics/ClassLength
|
144
146
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'cgi'
|
2
4
|
require 'fileutils'
|
3
5
|
require 'pathname'
|
@@ -6,6 +8,7 @@ module RServiceBus2
|
|
6
8
|
# Monitor for Directory
|
7
9
|
class MonitorDirNotifier < Monitor
|
8
10
|
attr_reader :path, :processing_folder, :filter
|
11
|
+
|
9
12
|
def connect(uri)
|
10
13
|
# Pass the path through the Dir object to check syntax on startup
|
11
14
|
begin
|
@@ -45,7 +48,7 @@ module RServiceBus2
|
|
45
48
|
begin
|
46
49
|
open_folder processing_uri.path
|
47
50
|
unless File.writable?(processing_uri.path)
|
48
|
-
puts "*****
|
51
|
+
puts "***** 1Processing Directory is not writable,
|
49
52
|
#{processing_uri.path}."
|
50
53
|
puts "***** Make the directory, #{processing_uri.path},
|
51
54
|
writable and try again."
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'rservicebus2/monitor'
|
2
4
|
require 'rservicebus2/monitor/message'
|
3
5
|
|
@@ -16,56 +18,64 @@ module RServiceBus2
|
|
16
18
|
@resource_list = {}
|
17
19
|
end
|
18
20
|
|
21
|
+
def conditionally_set_app_resource(monitor, key, val)
|
22
|
+
monitor.instance_variable_set("@#{key}", val.resource)
|
23
|
+
@resource_list[monitor.class.name] = [] if @resource_list[monitor.class.name].nil?
|
24
|
+
@resource_list[monitor.class.name] << val
|
25
|
+
@host.log "App resource attribute, #{key}, set for: #{monitor.class.name}"
|
26
|
+
end
|
27
|
+
|
19
28
|
# Assigns appropriate resources to writable attributes in the handler that
|
20
29
|
# match keys in the resource hash
|
21
30
|
# @param [RServiceBus2::Handler] handler
|
22
|
-
def
|
31
|
+
def conditionally_set_app_resources(monitor)
|
23
32
|
RServiceBus2.rlog "Checking app resources for: #{monitor.class.name}"
|
24
|
-
RServiceBus2.rlog "If your attribute is not getting set, check that it is
|
25
|
-
in the 'attr_accessor' list"
|
33
|
+
RServiceBus2.rlog "If your attribute is not getting set, check that it is in the 'attr_accessor' list"
|
26
34
|
@resource_manager.all.each do |k, v|
|
27
35
|
next unless monitor.class.method_defined?(k)
|
28
36
|
|
29
|
-
monitor
|
30
|
-
@resource_list[monitor.class.name] = [] if
|
31
|
-
@resource_list[monitor.class.name].nil?
|
32
|
-
@resource_list[monitor.class.name] << v
|
33
|
-
@host.log "App resource attribute, #{k}, set for: " +
|
34
|
-
monitor.class.name
|
37
|
+
conditionally_set_app_resource(monitor, k, v)
|
35
38
|
end
|
36
39
|
self
|
37
40
|
end
|
38
41
|
|
39
|
-
|
40
|
-
|
42
|
+
# rubocop:disable Metrics/MethodLength
|
43
|
+
def load_monitor(key, val)
|
44
|
+
name = key.sub('RSBOB_', '')
|
45
|
+
uri = URI.parse(val)
|
46
|
+
monitor = nil
|
47
|
+
case uri.scheme
|
48
|
+
when 'dir'
|
49
|
+
require 'rservicebus2/monitor/dir'
|
50
|
+
monitor = MonitorDir.new(@host, name, uri)
|
51
|
+
when 'awss3'
|
52
|
+
require 'rservicebus2/monitor/awss3'
|
53
|
+
monitor = MonitorAWSS3.new(@host, name, uri)
|
54
|
+
when 'awssqs'
|
55
|
+
require 'rservicebus2/monitor/awssqs'
|
56
|
+
monitor = MonitorAWSSQS.new(@host, name, uri)
|
57
|
+
when 'dirnotifier'
|
58
|
+
require 'rservicebus2/monitor/dirnotifier'
|
59
|
+
monitor = MonitorDirNotifier.new(@host, name, uri)
|
60
|
+
else
|
61
|
+
abort("Scheme, #{uri.scheme}, not recognised when configuring
|
62
|
+
Monitor, #{key}=#{val}")
|
63
|
+
end
|
64
|
+
monitor
|
65
|
+
end
|
66
|
+
# rubocop:enable Metrics/MethodLength
|
67
|
+
|
68
|
+
def monitors(env)
|
69
|
+
list = []
|
41
70
|
|
42
71
|
env.each do |k, v|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
when 'dir'
|
49
|
-
require 'rservicebus2/monitor/dir'
|
50
|
-
monitor = MonitorDir.new(@host, name, uri)
|
51
|
-
when 'awss3'
|
52
|
-
require 'rservicebus2/monitor/awss3'
|
53
|
-
monitor = MonitorAWSS3.new(@host, name, uri)
|
54
|
-
when 'awssqs'
|
55
|
-
require 'rservicebus2/monitor/awssqs'
|
56
|
-
monitor = MonitorAWSSQS.new(@host, name, uri)
|
57
|
-
when 'dirnotifier'
|
58
|
-
require 'rservicebus2/monitor/dirnotifier'
|
59
|
-
monitor = MonitorDirNotifier.new(@host, name, uri)
|
60
|
-
else
|
61
|
-
abort("Scheme, #{uri.scheme}, not recognised when configuring
|
62
|
-
Monitor, #{k}=#{v}")
|
63
|
-
end
|
64
|
-
set_app_resources(monitor)
|
65
|
-
monitors << monitor
|
66
|
-
end
|
72
|
+
next unless v.is_a?(String) && k.start_with?('RSBOB_')
|
73
|
+
|
74
|
+
monitor = load_monitor(k, v)
|
75
|
+
conditionally_set_app_resources(monitor)
|
76
|
+
list << monitor
|
67
77
|
end
|
68
|
-
|
78
|
+
list
|
69
79
|
end
|
70
80
|
end
|
71
81
|
end
|
data/lib/rservicebus2/mq.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
4
|
|
3
5
|
module RServiceBus2
|
@@ -10,6 +12,7 @@ module RServiceBus2
|
|
10
12
|
class MQ
|
11
13
|
attr_reader :local_queue_name
|
12
14
|
|
15
|
+
# rubocop:disable Metrics/MethodLength
|
13
16
|
def self.get
|
14
17
|
mq_string = RServiceBus2.get_value('RSBMQ', 'beanstalk://localhost')
|
15
18
|
uri = URI.parse(mq_string)
|
@@ -34,31 +37,26 @@ module RServiceBus2
|
|
34
37
|
|
35
38
|
mq
|
36
39
|
end
|
40
|
+
# rubocop:enable Metrics/MethodLength
|
37
41
|
|
38
42
|
# Resources are attached, and are be specified using the URI syntax
|
39
43
|
# @param [URI] uri the type and location of queue,
|
40
44
|
# eg beanstalk://127.0.0.1/foo
|
41
45
|
# @param [Integer] timeout the amount of time to wait for a msg to arrive
|
46
|
+
|
47
|
+
# rubocop:disable Metrics/MethodLength
|
42
48
|
def initialize(uri)
|
43
|
-
|
44
|
-
@uri = uri
|
45
|
-
else
|
46
|
-
puts 'uri must be a valid URI'
|
47
|
-
abort
|
48
|
-
end
|
49
|
+
abort 'Paramter to mq must be a valid URI' unless uri.is_a? URI
|
49
50
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
@local_queue_name[0] = ''
|
55
|
-
end
|
51
|
+
@uri = uri
|
52
|
+
@local_queue_name = uri.path
|
53
|
+
@local_queue_name[0] = '' if @local_queue_name[0] == '/'
|
54
|
+
@local_queue_name = RServiceBus2.get_value('APPNAME', 'RServiceBus') if @local_queue_name == ''
|
56
55
|
|
57
56
|
if @local_queue_name == ''
|
58
|
-
puts
|
59
|
-
|
60
|
-
|
61
|
-
puts '*** the structure is scheme://host[:port]/queuename'
|
57
|
+
puts 'Queue name must be supplied ' \
|
58
|
+
"*** uri, #{uri}, needs to contain a queue name" \
|
59
|
+
'*** the structure is scheme://host[:port]/queuename'
|
62
60
|
abort
|
63
61
|
end
|
64
62
|
|
@@ -66,42 +64,41 @@ module RServiceBus2
|
|
66
64
|
connect(uri.host, uri.port)
|
67
65
|
subscribe(@local_queue_name)
|
68
66
|
end
|
67
|
+
# rubocop:enable Metrics/MethodLength
|
69
68
|
|
70
69
|
# Connect to the broker
|
71
70
|
# @param [String] host machine runnig the mq
|
72
71
|
# @param [String] port port the mq is running on
|
73
72
|
def connect(_host, _port)
|
74
|
-
|
73
|
+
raise 'Method, connect, needs to be implemented'
|
75
74
|
end
|
76
75
|
|
77
76
|
# Connect to the receiving queue
|
78
77
|
# @param [String] queuename name of the receiving queue
|
79
78
|
def subscribe(_queuename)
|
80
|
-
|
79
|
+
raise 'Method, subscribe, needs to be implemented'
|
81
80
|
end
|
82
81
|
|
83
82
|
# Get next msg from queue
|
84
83
|
def pop
|
85
|
-
|
84
|
+
raise 'Method, pop, needs to be implemented'
|
86
85
|
end
|
87
86
|
|
88
87
|
# "Commit" the pop
|
89
88
|
def ack
|
90
|
-
|
89
|
+
raise 'Method, ack, needs to be implemented'
|
91
90
|
end
|
92
91
|
|
93
92
|
# At least called in the Host rescue block, to ensure all network links are
|
94
93
|
# healthy
|
95
94
|
# @param [String] queue_name name of the queue to which the msg should be sent
|
96
95
|
# @param [String] msg msg to be sent
|
97
|
-
def send(
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
end
|
105
|
-
|
106
|
-
end
|
96
|
+
def send(_queue_name, _msg)
|
97
|
+
@connection.close
|
98
|
+
rescue StandardError => e
|
99
|
+
puts "AppResource. An error was raised while closing connection to, #{@uri}"
|
100
|
+
puts "Error: #{e.message}"
|
101
|
+
puts "Backtrace: #{e.backtrace}"
|
102
|
+
end
|
103
|
+
end
|
107
104
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'beanstalk-client'
|
2
4
|
require 'rservicebus2/mq'
|
3
5
|
|
@@ -34,6 +36,7 @@ module RServiceBus2
|
|
34
36
|
abort
|
35
37
|
end
|
36
38
|
end
|
39
|
+
# rubocop:enable Metrics/AbcSize,Metrics/MethodLength
|
37
40
|
|
38
41
|
def subscribe(queuename)
|
39
42
|
@beanstalk.watch(queuename)
|
@@ -45,6 +48,7 @@ module RServiceBus2
|
|
45
48
|
@job = @beanstalk.reserve @timeout
|
46
49
|
rescue StandardError => e
|
47
50
|
raise NoMsgToProcess if e.message == 'TIMED_OUT'
|
51
|
+
|
48
52
|
raise e
|
49
53
|
end
|
50
54
|
@job.body
|
@@ -63,7 +67,7 @@ module RServiceBus2
|
|
63
67
|
if msg.length > @max_job_size
|
64
68
|
puts '***Attempting to send a msg which will not fit on queue.'
|
65
69
|
puts "***Msg size, #{msg.length}, max msg size, #{@max_job_size}."
|
66
|
-
|
70
|
+
raise JobTooBigError, "Msg size, #{msg.length}, max msg size,
|
67
71
|
#{@max_job_size}"
|
68
72
|
end
|
69
73
|
@beanstalk.use(queue_name)
|