rservicebus2 0.2.15 → 0.2.21
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/return_messages_to_source_queue +2 -1
- data/lib/rservicebus2/agent.rb +1 -1
- data/lib/rservicebus2/appresource.rb +10 -4
- data/lib/rservicebus2/appresource/awsdynamodb.rb +3 -1
- data/lib/rservicebus2/appresource/awss3.rb +3 -1
- data/lib/rservicebus2/appresource/awssqs.rb +3 -1
- data/lib/rservicebus2/appresource/fluiddb2.rb +1 -1
- data/lib/rservicebus2/handler_loader.rb +10 -5
- data/lib/rservicebus2/helper_functions.rb +20 -0
- data/lib/rservicebus2/host.rb +25 -13
- data/lib/rservicebus2/message.rb +2 -2
- data/lib/rservicebus2/monitor.rb +3 -1
- data/lib/rservicebus2/monitor/beanstalk.rb +2 -2
- data/lib/rservicebus2/monitor/dir.rb +62 -52
- data/lib/rservicebus2/monitor/dirnotifier.rb +58 -66
- data/lib/rservicebus2/mq/aws.rb +26 -40
- data/lib/rservicebus2/mq/beanstalk.rb +6 -8
- data/lib/rservicebus2/mq/file.rb +14 -21
- data/lib/rservicebus2/mq/redis.rb +14 -20
- data/lib/rservicebus2/saga_storage/dir.rb +1 -1
- data/lib/rservicebus2/sendat_storage/file.rb +1 -1
- data/lib/rservicebus2/state_storage/dir.rb +1 -1
- data/lib/rservicebus2/subscription_storage/file.rb +2 -2
- data/lib/rservicebus2/transporter.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 577432efc03536a99119146208b8fd418bd7641fc467f59f38564f4fdb477ca0
|
4
|
+
data.tar.gz: 85ade6550d036a70c0346f6d15bbc5d5376f49724b94bac7f48397e4ecd02bba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df0fb5b6725e33976c7347765dceb6366c6b9c38bc8c215dee36ee984507e83ce1357861033a4389c38173ac02700df005314050bf3da740c5a6efc45c75a412
|
7
|
+
data.tar.gz: 80d4d9d00f7faa47c7b17b19ffe4a6e841b06b96952f52d3be3fe3df573780713685fbeff17dd5ceef6cb0d20fd9d34f01fea0c2545da20150683a8e0e4f3f89
|
@@ -13,7 +13,8 @@ def return_msg(beanstalk, job, request_nbr)
|
|
13
13
|
|
14
14
|
msg_name = payload.match('(\w*)', start_index)[1]
|
15
15
|
|
16
|
-
msg =
|
16
|
+
msg = RServiceBus2.safe_load(payload)
|
17
|
+
|
17
18
|
if msg.last_error_string.nil?
|
18
19
|
puts "*** Requested msg, #{request_nbr}, does not have a sourceQueue to
|
19
20
|
which it can be returned"
|
data/lib/rservicebus2/agent.rb
CHANGED
@@ -33,7 +33,7 @@ module RServiceBus2
|
|
33
33
|
finished
|
34
34
|
end
|
35
35
|
|
36
|
-
#
|
36
|
+
# Allow for cleanup, outside of transaction semantics
|
37
37
|
def finished
|
38
38
|
@connection.close
|
39
39
|
end
|
@@ -53,12 +53,18 @@ module RServiceBus2
|
|
53
53
|
end
|
54
54
|
|
55
55
|
# Transaction Semantics
|
56
|
-
def begin
|
56
|
+
def begin
|
57
|
+
RServiceBus2.rlog "#{self.class.name}. Default transaction.begin called"
|
58
|
+
end
|
57
59
|
|
58
60
|
# Transaction Semantics
|
59
|
-
def commit
|
61
|
+
def commit
|
62
|
+
RServiceBus2.rlog "#{self.class.name}. Default transaction.commit called"
|
63
|
+
end
|
60
64
|
|
61
65
|
# Transaction Semantics
|
62
|
-
def rollback
|
66
|
+
def rollback
|
67
|
+
RServiceBus2.rlog "#{self.class.name}. Default transaction.rollback called"
|
68
|
+
end
|
63
69
|
end
|
64
70
|
end
|
@@ -140,14 +140,19 @@ module RServiceBus2
|
|
140
140
|
next if file_path.end_with?('.')
|
141
141
|
|
142
142
|
msg_name = get_msg_name(file_path)
|
143
|
+
RServiceBus2.add_to_permitted_classes(
|
144
|
+
msg_name.gsub(/(?<=_|^)(\w)/) { Regexp.last_match(1).upcase }.gsub(/(?:_)(\w)/, '\1')
|
145
|
+
)
|
146
|
+
|
143
147
|
if File.directory?(file_path)
|
144
148
|
load_handlers_from_second_level_path(msg_name, file_path)
|
145
|
-
|
146
|
-
# Classify
|
147
|
-
handler_name = "message_handler_#{msg_name}"
|
148
|
-
.gsub(/(?<=_|^)(\w)/) { Regexp.last_match(1).upcase }.gsub(/(?:_)(\w)/, '\1')
|
149
|
-
load_handler(msg_name, file_path, handler_name)
|
149
|
+
next
|
150
150
|
end
|
151
|
+
|
152
|
+
# Classify
|
153
|
+
handler_name = "message_handler_#{msg_name}"
|
154
|
+
.gsub(/(?<=_|^)(\w)/) { Regexp.last_match(1).upcase }.gsub(/(?:_)(\w)/, '\1')
|
155
|
+
load_handler(msg_name, file_path, handler_name)
|
151
156
|
end
|
152
157
|
|
153
158
|
self
|
@@ -1,7 +1,27 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'csv'
|
4
|
+
|
3
5
|
# Helper functions
|
4
6
|
module RServiceBus2
|
7
|
+
DEFAULT_PERMITTED_CLASSES = 'RServiceBus2::Message,Time,UUIDTools::UUID,URI::Generic,URI::RFC3986_Parser,Symbol,Regexp'
|
8
|
+
|
9
|
+
def self.permitted_classes
|
10
|
+
permitted_classes_string = get_value('PERMITTED_CLASSES', DEFAULT_PERMITTED_CLASSES)
|
11
|
+
CSV.parse(permitted_classes_string)[0]
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.add_to_permitted_classes(string)
|
15
|
+
ENV['PERMITTED_CLASSES'] = permitted_classes
|
16
|
+
.push(string)
|
17
|
+
.uniq
|
18
|
+
.to_csv
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.safe_load(body)
|
22
|
+
YAML.safe_load(body, permitted_classes: permitted_classes)
|
23
|
+
end
|
24
|
+
|
5
25
|
def self.convert_dto_to_hash(obj)
|
6
26
|
hash = {}
|
7
27
|
obj.instance_variables.each do |var|
|
data/lib/rservicebus2/host.rb
CHANGED
@@ -162,7 +162,7 @@ module RServiceBus2
|
|
162
162
|
body = @mq.pop
|
163
163
|
begin
|
164
164
|
@stats.inc_total_processed
|
165
|
-
@msg =
|
165
|
+
@msg = RServiceBus2.safe_load(body)
|
166
166
|
case @msg.msg.class.name
|
167
167
|
when 'RServiceBus2::MessageSubscription'
|
168
168
|
@subscription_manager.add(@msg.msg.event_name,
|
@@ -252,18 +252,30 @@ module RServiceBus2
|
|
252
252
|
message_loop = false
|
253
253
|
rescue NoMsgToProcess => e
|
254
254
|
# This exception is just saying there are no messages to process
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
255
|
+
begin
|
256
|
+
@queue_for_msgs_to_be_sent_on_complete = []
|
257
|
+
@monitors.each(&:look)
|
258
|
+
send_queued_msgs
|
259
|
+
@queue_for_msgs_to_be_sent_on_complete = nil
|
260
|
+
|
261
|
+
@queue_for_msgs_to_be_sent_on_complete = []
|
262
|
+
@cron_manager.run
|
263
|
+
send_queued_msgs
|
264
|
+
@queue_for_msgs_to_be_sent_on_complete = nil
|
265
|
+
|
266
|
+
@send_at_manager.process
|
267
|
+
@circuit_breaker.success
|
268
|
+
rescue StandardError => e
|
269
|
+
if e.message == 'SIGTERM' || e.message == 'SIGINT'
|
270
|
+
puts 'Exiting on request ...'
|
271
|
+
message_loop = false
|
272
|
+
else
|
273
|
+
puts '*** This is really unexpected.'
|
274
|
+
message_loop = false
|
275
|
+
puts "Message: #{e.message}"
|
276
|
+
puts e.backtrace
|
277
|
+
end
|
278
|
+
end
|
267
279
|
rescue StandardError => e
|
268
280
|
if e.message == 'SIGTERM' || e.message == 'SIGINT'
|
269
281
|
puts 'Exiting on request ...'
|
data/lib/rservicebus2/message.rb
CHANGED
@@ -55,9 +55,9 @@ module RServiceBus2
|
|
55
55
|
|
56
56
|
# @return [Object] The msg to be sent
|
57
57
|
def msg
|
58
|
-
return
|
58
|
+
return RServiceBus2.safe_load(Zlib::Inflate.inflate(@_msg)) if @compressed == true
|
59
59
|
|
60
|
-
|
60
|
+
RServiceBus2.safe_load(@_msg)
|
61
61
|
rescue ArgumentError => e
|
62
62
|
raise e if e.message.index('undefined class/module ').nil?
|
63
63
|
|
data/lib/rservicebus2/monitor.rb
CHANGED
@@ -36,7 +36,9 @@ module RServiceBus2
|
|
36
36
|
end
|
37
37
|
|
38
38
|
# A notification that allows cleanup
|
39
|
-
def finished
|
39
|
+
def finished
|
40
|
+
RServiceBus2.rlog "#{self.class.name}. Default Finished"
|
41
|
+
end
|
40
42
|
|
41
43
|
# At least called in the Host rescue block, to ensure all network links
|
42
44
|
# are healthy
|
@@ -12,7 +12,7 @@ module RServiceBus2
|
|
12
12
|
CGI.parse(u.query)['timeout1']&.first || 5
|
13
13
|
end
|
14
14
|
|
15
|
-
# rubocop:disable Metrics/MethodLength
|
15
|
+
# rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
16
16
|
def connect(uri)
|
17
17
|
@uri = uri
|
18
18
|
@timeout = deduce_timeout(uri)
|
@@ -36,7 +36,7 @@ module RServiceBus2
|
|
36
36
|
puts e.message
|
37
37
|
puts e.backtrace
|
38
38
|
end
|
39
|
-
# rubocop:enable Metrics/MethodLength
|
39
|
+
# rubocop:enable Metrics/MethodLength,Metrics/AbcSize
|
40
40
|
|
41
41
|
def look
|
42
42
|
job = @beanstalk.reserve @timeout
|
@@ -9,6 +9,7 @@ module RServiceBus2
|
|
9
9
|
|
10
10
|
# rubocop:disable Metrics/ClassLength
|
11
11
|
class MonitorDir < Monitor
|
12
|
+
# rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
12
13
|
def input_dir(uri)
|
13
14
|
# Pass the path through the Dir object to check syntax on startup
|
14
15
|
return Dir.new(uri.path) if File.writable?(uri.path)
|
@@ -27,44 +28,48 @@ module RServiceBus2
|
|
27
28
|
"***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
28
29
|
abort
|
29
30
|
end
|
31
|
+
# rubocop:enable Metrics/MethodLength,Metrics/AbcSize
|
30
32
|
|
31
|
-
def
|
32
|
-
|
33
|
-
@input_filter = []
|
34
|
-
|
35
|
-
return if uri.query.nil?
|
33
|
+
def initialise_archive(parts)
|
34
|
+
return unless parts.key?('archive')
|
36
35
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
puts '***** Archive file name templating not yet supported.'
|
43
|
-
puts "***** Directory's only."
|
44
|
-
abort
|
45
|
-
end
|
46
|
-
@archivedir = archiveuri.path
|
36
|
+
archiveuri = URI.parse(parts['archive'][0])
|
37
|
+
unless File.directory?(archiveuri.path)
|
38
|
+
puts '***** Archive file name templating not yet supported.'
|
39
|
+
puts "***** Directory's only."
|
40
|
+
abort
|
47
41
|
end
|
42
|
+
@archivedir = archiveuri.path
|
43
|
+
end
|
48
44
|
|
49
|
-
|
50
|
-
|
45
|
+
def initialise_input_filter(parts)
|
51
46
|
if parts['input_filter'].count > 1
|
52
|
-
puts 'Too many input_filters specified.'
|
53
|
-
puts '*** ZIP, or GZ are the only valid input_filters.'
|
47
|
+
puts 'Too many input_filters specified.\n*** ZIP, or GZ are the only valid input_filters.'
|
54
48
|
abort
|
55
49
|
end
|
56
50
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
else
|
61
|
-
puts 'Invalid input_filter specified.'
|
62
|
-
puts '*** ZIP, or GZ are the only valid input_filters.'
|
51
|
+
unless %w[ZIP GZ TAR].include?(parts['input_filter'][0]).nil?
|
52
|
+
puts 'Invalid input_filter specified.\n' \
|
53
|
+
'*** ZIP, or GZ are the only valid input_filters.'
|
63
54
|
abort
|
64
55
|
end
|
56
|
+
|
65
57
|
@input_filter << parts['input_filter'][0]
|
66
58
|
end
|
67
59
|
|
60
|
+
def connect(uri)
|
61
|
+
@path = input_dir(uri).path
|
62
|
+
@input_filter = []
|
63
|
+
|
64
|
+
return if uri.query.nil?
|
65
|
+
|
66
|
+
parts = CGI.parse(uri.query)
|
67
|
+
@querystringparts = parts
|
68
|
+
initialise_archive(parts)
|
69
|
+
|
70
|
+
initialise_input_filter(parts) if parts.key?('input_filter')
|
71
|
+
end
|
72
|
+
|
68
73
|
def process_content(content)
|
69
74
|
content
|
70
75
|
end
|
@@ -86,13 +91,14 @@ module RServiceBus2
|
|
86
91
|
# rubocop:disable Metrics/MethodLength
|
87
92
|
def read_content_from_file(file_path)
|
88
93
|
content = ''
|
89
|
-
if @input_filter.length
|
90
|
-
|
94
|
+
if @input_filter.length.positive?
|
95
|
+
case @input_filter[0]
|
96
|
+
when 'ZIP'
|
91
97
|
content = read_content_from_zip_file(file_path)
|
92
|
-
|
98
|
+
when 'GZ'
|
93
99
|
content = read_content_from_gz_file(file_path)
|
94
|
-
|
95
|
-
|
100
|
+
else
|
101
|
+
raise "#{@input_filter[0]} reader not implemented."
|
96
102
|
end
|
97
103
|
|
98
104
|
else
|
@@ -101,13 +107,34 @@ module RServiceBus2
|
|
101
107
|
|
102
108
|
content
|
103
109
|
end
|
110
|
+
# rubocop:enable Metrics/MethodLength
|
111
|
+
|
112
|
+
def archive_file(file_path, content)
|
113
|
+
basename = File.basename(file_path)
|
114
|
+
new_file_path = "#{@archivedir}/#{basename}.#{Time.now.strftime('%Y%m%d%H%M%S%L')}.zip"
|
115
|
+
RServiceBus2.log "Writing to archive, #{new_file_path}"
|
116
|
+
|
117
|
+
Zip::ZipOutputStream.open(new_file_path) do |zos|
|
118
|
+
zos.put_next_entry(basename)
|
119
|
+
zos.puts content
|
120
|
+
end
|
121
|
+
end
|
104
122
|
|
105
123
|
def process_path(file_path)
|
124
|
+
if File.file?(file_path) != true
|
125
|
+
RServiceBus2.log "Skipping directory, #{file_path}"
|
126
|
+
return
|
127
|
+
end
|
128
|
+
|
129
|
+
RServiceBus2.log "Ready to process, #{file_path}"
|
106
130
|
content = read_content_from_file(file_path)
|
107
131
|
payload = process_content(content)
|
108
132
|
|
109
|
-
send(payload, URI.parse(
|
110
|
-
|
133
|
+
send(payload, URI.parse(CGI.escape("file://#{file_path}")))
|
134
|
+
|
135
|
+
archive_file(file_path, content) unless @archivedir.nil?
|
136
|
+
|
137
|
+
File.unlink(file_path)
|
111
138
|
end
|
112
139
|
|
113
140
|
def look
|
@@ -116,28 +143,11 @@ module RServiceBus2
|
|
116
143
|
|
117
144
|
file_list = Dir.glob("#{@path}/*")
|
118
145
|
file_list.each do |file_path|
|
119
|
-
|
120
|
-
RServiceBus2.log "Skipping directory, #{file_path}"
|
121
|
-
next
|
122
|
-
end
|
123
|
-
RServiceBus2.log "Ready to process, #{file_path}"
|
124
|
-
content = process_path(file_path)
|
125
|
-
|
126
|
-
unless @archivedir.nil?
|
127
|
-
basename = File.basename(file_path)
|
128
|
-
new_file_path = "#{@archivedir}/#{basename}.#{Time.now.strftime('%Y%m%d%H%M%S%L')}.zip"
|
129
|
-
RServiceBus2.log "Writing to archive, #{new_file_path}"
|
130
|
-
|
131
|
-
Zip::ZipOutputStream.open(new_file_path) do |zos|
|
132
|
-
zos.put_next_entry(basename)
|
133
|
-
zos.puts content
|
134
|
-
end
|
135
|
-
end
|
136
|
-
File.unlink(file_path)
|
146
|
+
process_path(file_path)
|
137
147
|
|
138
148
|
file_processed += 1
|
139
|
-
RServiceBus2.log "Processed #{file_processed} of #{file_list.length}."
|
140
|
-
|
149
|
+
RServiceBus2.log "Processed #{file_processed} of #{file_list.length}.\nAllow system tick #{self.class.name}"
|
150
|
+
|
141
151
|
break if file_processed >= max_files_processed
|
142
152
|
end
|
143
153
|
end
|
@@ -9,78 +9,70 @@ module RServiceBus2
|
|
9
9
|
class MonitorDirNotifier < Monitor
|
10
10
|
attr_reader :path, :processing_folder, :filter
|
11
11
|
|
12
|
-
def
|
13
|
-
#
|
14
|
-
|
15
|
-
|
16
|
-
unless File.writable?(uri.path)
|
17
|
-
puts "***** Directory is not writable, #{uri.path}."
|
18
|
-
puts "***** Make the directory, #{uri.path}, writable and try again."
|
19
|
-
abort
|
20
|
-
end
|
21
|
-
rescue Errno::ENOENT
|
22
|
-
puts "***** Directory does not exist, #{uri.path}."
|
23
|
-
puts "***** Create the directory, #{uri.path}, and try again."
|
24
|
-
puts "***** eg, mkdir #{uri.path}"
|
25
|
-
abort
|
26
|
-
rescue Errno::ENOTDIR
|
27
|
-
puts "***** The specified path does not point to a directory,
|
28
|
-
#{uri.path}."
|
29
|
-
puts "***** Either repoint path to a directory, or remove, #{uri.path},
|
30
|
-
and create it as a directory."
|
31
|
-
puts "***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
32
|
-
abort
|
33
|
-
end
|
12
|
+
def directory_not_writable(path, param_name)
|
13
|
+
"***** #{param_name} is not writable, #{path}.\n" \
|
14
|
+
"***** Make the directory, #{path}, writable and try again."
|
15
|
+
end
|
34
16
|
|
35
|
-
|
17
|
+
def directory_does_not_exist(path, param_name)
|
18
|
+
"***** #{param_name} does not exist, #{path}.\n" \
|
19
|
+
"***** Create the directory, #{path}, and try again.\n" \
|
20
|
+
"***** eg, mkdir #{path}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def path_is_not_a_directory(path)
|
24
|
+
"***** The specified path does not point to a directory, #{path}.\n" \
|
25
|
+
"***** Either repoint path to a directory, or remove, #{path}, and create it as a directory.\n" \
|
26
|
+
"***** eg, rm #{path} && mkdir #{path}"
|
27
|
+
end
|
28
|
+
|
29
|
+
def processing_directory_not_specified(path)
|
30
|
+
'***** Processing Directory is not specified.' \
|
31
|
+
'***** Specify the Processing Directory as a query string in the Path URI' \
|
32
|
+
"***** eg, '/#{path}?processing=*ProcessingDir*'"
|
33
|
+
end
|
36
34
|
|
35
|
+
def validate_directory(path, param_name)
|
36
|
+
open_folder path
|
37
|
+
return if File.writable?(path)
|
38
|
+
|
39
|
+
puts directory_not_writable(path, param_name)
|
40
|
+
abort
|
41
|
+
rescue Errno::ENOENT
|
42
|
+
puts directory_does_not_exist(path, param_name)
|
43
|
+
abort
|
44
|
+
rescue Errno::ENOTDIR
|
45
|
+
puts path_is_not_a_directory(path)
|
46
|
+
abort
|
47
|
+
end
|
48
|
+
|
49
|
+
def validate_processing_directory(uri)
|
37
50
|
if uri.query.nil?
|
38
|
-
puts
|
39
|
-
puts '***** Specify the Processing Directory as a query string in the
|
40
|
-
Path URI'
|
41
|
-
puts "***** eg, '/#{uri.path}?processing=*ProcessingDir*"
|
51
|
+
puts processing_directory_not_specified(uri.path)
|
42
52
|
abort
|
43
|
-
else
|
44
|
-
parts = CGI.parse(uri.query)
|
45
|
-
|
46
|
-
if parts.key? 'processing'
|
47
|
-
processing_uri = URI.parse parts['processing'][0]
|
48
|
-
begin
|
49
|
-
open_folder processing_uri.path
|
50
|
-
unless File.writable?(processing_uri.path)
|
51
|
-
puts "***** 1Processing Directory is not writable,
|
52
|
-
#{processing_uri.path}."
|
53
|
-
puts "***** Make the directory, #{processing_uri.path},
|
54
|
-
writable and try again."
|
55
|
-
abort
|
56
|
-
end
|
57
|
-
rescue Errno::ENOENT
|
58
|
-
puts "***** Processing Directory does not exist,
|
59
|
-
#{processing_uri.path}."
|
60
|
-
puts "***** Create the directory, #{processing_uri.path}, and try
|
61
|
-
again."
|
62
|
-
puts "***** eg, mkdir #{processing_uri.path}"
|
63
|
-
abort
|
64
|
-
rescue Errno::ENOTDIR
|
65
|
-
puts "***** Processing Directory does not point to a directory,
|
66
|
-
#{processing_uri.path}."
|
67
|
-
puts "***** Either repoint path to a directory, or remove,
|
68
|
-
#{processing_uri.path}, and create it as a directory."
|
69
|
-
puts "***** eg, rm #{processing_uri.path} && mkdir
|
70
|
-
#{processing_uri.path}"
|
71
|
-
abort
|
72
|
-
end
|
73
|
-
|
74
|
-
@processing_folder = processing_uri.path
|
75
|
-
end
|
76
|
-
|
77
|
-
@filter = '*'
|
78
|
-
@filter = parts['filter'][0] if parts.key? 'filter'
|
79
53
|
end
|
54
|
+
|
55
|
+
parts = CGI.parse(uri.query)
|
56
|
+
return unless parts.key? 'processing'
|
57
|
+
|
58
|
+
processing_uri = URI.parse parts['processing'][0]
|
59
|
+
validate_directory processing_uri.path, 'Processing Directory'
|
60
|
+
@processing_folder = processing_uri.path
|
61
|
+
end
|
62
|
+
|
63
|
+
def connect(uri)
|
64
|
+
# Pass the path through the Dir object to check syntax on startup
|
65
|
+
|
66
|
+
validate_directory uri, 'Directory'
|
67
|
+
@path = uri.path
|
68
|
+
validate_processing_directory(uri)
|
69
|
+
|
70
|
+
@filter = '*'
|
71
|
+
@filter = parts['filter'][0] if parts.key? 'filter'
|
80
72
|
end
|
81
73
|
|
82
74
|
def look
|
83
|
-
file_list =
|
75
|
+
file_list = files
|
84
76
|
file_list.each do |file_path|
|
85
77
|
new_path = move_file(file_path, @processing_folder)
|
86
78
|
send(nil, URI.parse("file://#{new_path}"))
|
@@ -97,8 +89,8 @@ module RServiceBus2
|
|
97
89
|
Pathname.new(dest).join(filename)
|
98
90
|
end
|
99
91
|
|
100
|
-
def
|
101
|
-
Dir.glob(Pathname.new(
|
92
|
+
def files
|
93
|
+
Dir.glob(Pathname.new(@path).join(@filter)).select { |f| File.file?(f) }
|
102
94
|
end
|
103
95
|
end
|
104
96
|
end
|
data/lib/rservicebus2/mq/aws.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sdk-sqs'
|
2
4
|
require 'aws-sdk-sts'
|
3
5
|
require 'rservicebus2/mq'
|
@@ -6,75 +8,59 @@ module RServiceBus2
|
|
6
8
|
# Beanstalk client implementation.
|
7
9
|
class MQAWS < MQ
|
8
10
|
# Connect to the broker
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
@max_job_size = 4_194_304
|
13
|
-
@region = region
|
11
|
+
def connect(region, _port)
|
12
|
+
@max_job_size = 4_194_304
|
13
|
+
@region = region
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end
|
15
|
+
sts_client = Aws::STS::Client.new(region: region)
|
16
|
+
@caller_identity_account = sts_client.get_caller_identity.account
|
17
|
+
rescue StandardError => e
|
18
|
+
puts 'Error connecting to AWS'
|
19
|
+
puts "Host string, #{region}"
|
20
|
+
puts e.message
|
21
|
+
puts e.backtrace
|
22
|
+
abort
|
24
23
|
end
|
25
24
|
|
26
25
|
def subscribe(queuename)
|
27
26
|
# For example:
|
28
27
|
# 'https://sqs.us-east-1.amazonaws.com/111111111111/my-queue'
|
29
|
-
@queue_url =
|
30
|
-
@caller_identity_account + '/' + queuename
|
28
|
+
@queue_url = "https://sqs.#{@region}.amazonaws.com/#{@caller_identity_account}/#{queuename}"
|
31
29
|
@sqs_client = Aws::SQS::Client.new(region: @region)
|
32
30
|
end
|
33
31
|
|
34
32
|
# Get next msg from queue
|
35
33
|
def pop
|
36
|
-
|
37
|
-
response = @sqs_client.receive_message(
|
38
|
-
queue_url: @queue_url,
|
39
|
-
max_number_of_messages: 1
|
40
|
-
)
|
34
|
+
response = @sqs_client.receive_message(queue_url: @queue_url, max_number_of_messages: 1)
|
41
35
|
|
42
|
-
|
43
|
-
raise NoMsgToProcess
|
44
|
-
end
|
36
|
+
raise NoMsgToProcess if response.messages.count.zero?
|
45
37
|
|
46
|
-
|
47
|
-
|
48
|
-
end
|
49
|
-
rescue StandardError => e
|
50
|
-
raise e
|
38
|
+
response.messages.each do |message|
|
39
|
+
@job = message
|
51
40
|
end
|
41
|
+
rescue StandardError => e
|
42
|
+
raise e
|
43
|
+
ensure
|
52
44
|
@job.body
|
53
45
|
end
|
54
46
|
|
55
47
|
def return_to_queue
|
56
|
-
# @job.release if @job
|
57
48
|
@job = nil
|
58
49
|
end
|
59
50
|
|
60
51
|
def ack
|
61
|
-
@sqs_client.delete_message({
|
62
|
-
queue_url: @queue_url,
|
63
|
-
receipt_handle: @job.receipt_handle
|
64
|
-
})
|
52
|
+
@sqs_client.delete_message({ queue_url: @queue_url, receipt_handle: @job.receipt_handle })
|
65
53
|
@job = nil
|
66
54
|
end
|
67
55
|
|
68
56
|
def send(queue_name, msg)
|
69
57
|
if msg.length > @max_job_size
|
70
|
-
puts '***Attempting to send a msg which will not fit on queue.'
|
71
|
-
|
72
|
-
|
73
|
-
#{@max_job_size}"
|
58
|
+
puts '***Attempting to send a msg which will not fit on queue.' \
|
59
|
+
"***Msg size, #{msg.length}, max msg size, #{@max_job_size}."
|
60
|
+
raise JobTooBigError, "Msg size, #{msg.length}, max msg size, #{@max_job_size}"
|
74
61
|
end
|
75
62
|
|
76
|
-
queue_url =
|
77
|
-
@caller_identity_account + '/' + queue_name
|
63
|
+
queue_url = "https://sqs.#{@region}.amazonaws.com/#{@caller_identity_account}/#{queue_name}"
|
78
64
|
|
79
65
|
@sqs_client.send_message(
|
80
66
|
queue_url: queue_url,
|
@@ -16,19 +16,17 @@ module RServiceBus2
|
|
16
16
|
@beanstalk = Beanstalk::Pool.new([string])
|
17
17
|
@max_job_size = @beanstalk.stats['max-job-size']
|
18
18
|
if @max_job_size < 4_194_304
|
19
|
-
puts
|
20
|
-
max-job-size, #{@max_job_size.to_f / (1024 * 1024)}m"
|
21
|
-
puts '***WARNING: Set the job size with the -z switch, eg
|
22
|
-
/usr/local/bin/beanstalkd -z 4194304'
|
19
|
+
puts '***WARNING: Lowest recommended.max-job-size is 4m, current ' \
|
20
|
+
"max-job-size, #{@max_job_size.to_f / (1024 * 1024)}m"
|
21
|
+
puts '***WARNING: Set the job size with the -z switch, eg ' \
|
22
|
+
'/usr/local/bin/beanstalkd -z 4194304'
|
23
23
|
end
|
24
24
|
rescue StandardError => e
|
25
25
|
puts 'Error connecting to Beanstalk'
|
26
26
|
puts "Host string, #{string}"
|
27
27
|
if e.message == 'Beanstalk::NotConnected'
|
28
|
-
puts '***Most likely, beanstalk is not running. Start beanstalk,
|
29
|
-
|
30
|
-
puts "***If you still get this error, check beanstalk is running
|
31
|
-
at, #{string}"
|
28
|
+
puts '***Most likely, beanstalk is not running. Start beanstalk, and try running this again.'
|
29
|
+
puts "***If you still get this error, check beanstalk is running at, #{string}"
|
32
30
|
else
|
33
31
|
puts e.message
|
34
32
|
puts e.backtrace
|
data/lib/rservicebus2/mq/file.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'fileutils'
|
2
4
|
require 'rservicebus2/mq'
|
3
5
|
|
@@ -5,18 +7,10 @@ module RServiceBus2
|
|
5
7
|
# Beanstalk client implementation.
|
6
8
|
class MQFile < MQ
|
7
9
|
# Connect to the broker
|
8
|
-
|
9
|
-
|
10
|
+
|
10
11
|
def initialize(uri)
|
11
|
-
|
12
|
-
@uri = uri
|
13
|
-
else
|
14
|
-
puts 'uri must be a valid URI'
|
15
|
-
abort
|
16
|
-
end
|
12
|
+
super
|
17
13
|
|
18
|
-
FileUtils.mkdir_p(@uri.path)
|
19
|
-
@local_queue_name = RServiceBus2.get_value('APPNAME', 'RServiceBus')
|
20
14
|
FileUtils.mkdir_p("#{@uri.path}/#{@local_queue_name}")
|
21
15
|
@timeout = RServiceBus2.get_value('QUEUE_TIMEOUT', '5').to_i
|
22
16
|
end
|
@@ -28,24 +22,23 @@ module RServiceBus2
|
|
28
22
|
end
|
29
23
|
|
30
24
|
# Get next msg from queue
|
25
|
+
# rubocop:disable Metrics/MethodLength
|
31
26
|
def pop
|
32
27
|
time = @timeout
|
33
|
-
while time
|
28
|
+
while time.positive?
|
34
29
|
files = Dir.glob("#{@uri.path}/#{@local_queue_name}/*.msg")
|
35
|
-
if files.
|
36
|
-
@file_path = files[0]
|
30
|
+
if files.positive?
|
31
|
+
@file_path = files[0]
|
37
32
|
@body = IO.read(@file_path)
|
38
|
-
|
33
|
+
return @body
|
39
34
|
end
|
40
|
-
|
41
|
-
|
35
|
+
time -= 1
|
36
|
+
sleep(1)
|
42
37
|
end
|
43
|
-
|
44
|
-
raise NoMsgToProcess if files.length == 0
|
45
|
-
end
|
46
38
|
|
47
|
-
|
39
|
+
raise NoMsgToProcess if files.length.zero?
|
48
40
|
end
|
41
|
+
# rubocop:enable Metrics/MethodLength
|
49
42
|
|
50
43
|
def ack
|
51
44
|
FileUtils.rm @file_path
|
@@ -53,7 +46,7 @@ module RServiceBus2
|
|
53
46
|
|
54
47
|
def send(queue_name, msg)
|
55
48
|
FileUtils.mkdir_p("#{@uri.path}/#{queue_name}")
|
56
|
-
IO.write("#{@uri.path}/#{queue_name}/#{rand
|
49
|
+
IO.write("#{@uri.path}/#{queue_name}/#{rand}.msg", msg)
|
57
50
|
end
|
58
51
|
end
|
59
52
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'redis'
|
2
4
|
require 'rservicebus2/mq'
|
3
5
|
|
@@ -5,38 +7,31 @@ module RServiceBus2
|
|
5
7
|
# Redis client implementation.
|
6
8
|
class MQRedis < MQ
|
7
9
|
# Connect to the broker
|
8
|
-
# rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
9
10
|
def connect(host, port)
|
10
11
|
port ||= 6379
|
11
12
|
string = "#{host}:#{port}"
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
puts "Host string, #{string}"
|
20
|
-
abort()
|
21
|
-
end
|
14
|
+
@redis = Redis.new(host: host, port: port)
|
15
|
+
rescue StandardError => e
|
16
|
+
puts e.message
|
17
|
+
puts 'Error connecting to Redis for mq'
|
18
|
+
puts "Host string, #{string}"
|
19
|
+
abort
|
22
20
|
end
|
23
21
|
|
24
22
|
# Connect to the queue
|
25
|
-
def subscribe(
|
23
|
+
def subscribe(queuename)
|
26
24
|
@queuename = queuename
|
27
25
|
end
|
28
26
|
|
29
27
|
# Get next msg from queue
|
30
28
|
def pop
|
31
|
-
if @redis.llen(
|
29
|
+
if @redis.llen(@queuename).zero?
|
32
30
|
sleep @timeout
|
33
|
-
raise NoMsgToProcess
|
31
|
+
raise NoMsgToProcess
|
34
32
|
end
|
35
33
|
|
36
|
-
|
37
|
-
end
|
38
|
-
|
39
|
-
def returnToQueue
|
34
|
+
@redis.lindex @queuename, 0
|
40
35
|
end
|
41
36
|
|
42
37
|
# "Commit" queue
|
@@ -45,9 +40,8 @@ module RServiceBus2
|
|
45
40
|
end
|
46
41
|
|
47
42
|
# At least called in the Host rescue block, to ensure all network links are healthy
|
48
|
-
def send(
|
49
|
-
@redis.rpush
|
43
|
+
def send(queue_name, msg)
|
44
|
+
@redis.rpush queue_name, msg
|
50
45
|
end
|
51
|
-
|
52
46
|
end
|
53
47
|
end
|
@@ -7,11 +7,11 @@ module RServiceBus2
|
|
7
7
|
RServiceBus2.log 'Load subscriptions'
|
8
8
|
return {} unless File.exist?(@uri.path)
|
9
9
|
|
10
|
-
|
10
|
+
RServiceBus2.safe_load(File.open(@uri.path))
|
11
11
|
end
|
12
12
|
|
13
13
|
def add(event_name, queue_name)
|
14
|
-
s = File.exist?(@uri.path) ?
|
14
|
+
s = File.exist?(@uri.path) ? RServiceBus2.safe_load(File.open(@uri.path)) : {}
|
15
15
|
s[event_name] = [] if s[event_name].nil?
|
16
16
|
|
17
17
|
s[event_name] << queue_name
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rservicebus2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Guy Irvine
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-05-
|
11
|
+
date: 2021-05-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: beanstalk-client
|