rservicebus2 0.2.13 → 0.2.19
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rservicebus2/appresource.rb +10 -4
- data/lib/rservicebus2/appresource/awsdynamodb.rb +3 -1
- data/lib/rservicebus2/appresource/awss3.rb +3 -1
- data/lib/rservicebus2/appresource/awssqs.rb +3 -1
- data/lib/rservicebus2/appresource/fluiddb2.rb +1 -1
- data/lib/rservicebus2/host.rb +24 -12
- data/lib/rservicebus2/monitor.rb +3 -1
- data/lib/rservicebus2/monitor/beanstalk.rb +53 -0
- data/lib/rservicebus2/monitor/dir.rb +62 -52
- data/lib/rservicebus2/monitor/dirnotifier.rb +40 -64
- data/lib/rservicebus2/monitor_configure.rb +3 -0
- data/lib/rservicebus2/mq/aws.rb +26 -40
- data/lib/rservicebus2/mq/beanstalk.rb +6 -8
- data/lib/rservicebus2/mq/file.rb +14 -21
- data/lib/rservicebus2/mq/redis.rb +14 -20
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 40740ce5d75540b3c751035c3b36e9829c14cae0decfaafeaca9ea02d59c775d
|
4
|
+
data.tar.gz: 52ad2edd143b3b5248a44510412265d86fd64c16cd1777a69d970203ba5be239
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: af23a4c627e06a151acee10fc4a1be2582195d0df9f7d38bd127c446802a79dbdbb0e82cf9a22a9d2df2ffd572b6e1654bb42706eb5a2bf4a6c1f4b66d336b26
|
7
|
+
data.tar.gz: 27003eec5f598361137ee0d187a4432fd91e1f1a2c19aebd724ae727adb47cc8018ec25ecd2e0d339d5acefe52fd4035646c52992374a7bf15cd3ffa1cf8643c
|
@@ -33,7 +33,7 @@ module RServiceBus2
|
|
33
33
|
finished
|
34
34
|
end
|
35
35
|
|
36
|
-
#
|
36
|
+
# Allow for cleanup, outside of transaction semantics
|
37
37
|
def finished
|
38
38
|
@connection.close
|
39
39
|
end
|
@@ -53,12 +53,18 @@ module RServiceBus2
|
|
53
53
|
end
|
54
54
|
|
55
55
|
# Transaction Semantics
|
56
|
-
def begin
|
56
|
+
def begin
|
57
|
+
RServiceBus2.rlog "#{self.class.name}. Default transaction.begin called"
|
58
|
+
end
|
57
59
|
|
58
60
|
# Transaction Semantics
|
59
|
-
def commit
|
61
|
+
def commit
|
62
|
+
RServiceBus2.rlog "#{self.class.name}. Default transaction.commit called"
|
63
|
+
end
|
60
64
|
|
61
65
|
# Transaction Semantics
|
62
|
-
def rollback
|
66
|
+
def rollback
|
67
|
+
RServiceBus2.rlog "#{self.class.name}. Default transaction.rollback called"
|
68
|
+
end
|
63
69
|
end
|
64
70
|
end
|
data/lib/rservicebus2/host.rb
CHANGED
@@ -252,18 +252,30 @@ module RServiceBus2
|
|
252
252
|
message_loop = false
|
253
253
|
rescue NoMsgToProcess => e
|
254
254
|
# This exception is just saying there are no messages to process
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
255
|
+
begin
|
256
|
+
@queue_for_msgs_to_be_sent_on_complete = []
|
257
|
+
@monitors.each(&:look)
|
258
|
+
send_queued_msgs
|
259
|
+
@queue_for_msgs_to_be_sent_on_complete = nil
|
260
|
+
|
261
|
+
@queue_for_msgs_to_be_sent_on_complete = []
|
262
|
+
@cron_manager.run
|
263
|
+
send_queued_msgs
|
264
|
+
@queue_for_msgs_to_be_sent_on_complete = nil
|
265
|
+
|
266
|
+
@send_at_manager.process
|
267
|
+
@circuit_breaker.success
|
268
|
+
rescue StandardError => e
|
269
|
+
if e.message == 'SIGTERM' || e.message == 'SIGINT'
|
270
|
+
puts 'Exiting on request ...'
|
271
|
+
message_loop = false
|
272
|
+
else
|
273
|
+
puts '*** This is really unexpected.'
|
274
|
+
message_loop = false
|
275
|
+
puts "Message: #{e.message}"
|
276
|
+
puts e.backtrace
|
277
|
+
end
|
278
|
+
end
|
267
279
|
rescue StandardError => e
|
268
280
|
if e.message == 'SIGTERM' || e.message == 'SIGINT'
|
269
281
|
puts 'Exiting on request ...'
|
data/lib/rservicebus2/monitor.rb
CHANGED
@@ -36,7 +36,9 @@ module RServiceBus2
|
|
36
36
|
end
|
37
37
|
|
38
38
|
# A notification that allows cleanup
|
39
|
-
def finished
|
39
|
+
def finished
|
40
|
+
RServiceBus2.rlog "#{self.class.name}. Default Finished"
|
41
|
+
end
|
40
42
|
|
41
43
|
# At least called in the Host rescue block, to ensure all network links
|
42
44
|
# are healthy
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'beanstalk-client'
|
4
|
+
require 'cgi'
|
5
|
+
|
6
|
+
module RServiceBus2
|
7
|
+
# Monitor S3 Bucket for objects
|
8
|
+
class MonitorBeanstalk < Monitor
|
9
|
+
def deduce_timeout(uri)
|
10
|
+
return 5 if uri.query.nil?
|
11
|
+
|
12
|
+
CGI.parse(u.query)['timeout1']&.first || 5
|
13
|
+
end
|
14
|
+
|
15
|
+
# rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
16
|
+
def connect(uri)
|
17
|
+
@uri = uri
|
18
|
+
@timeout = deduce_timeout(uri)
|
19
|
+
|
20
|
+
queue_name = uri.path.sub('/', '')
|
21
|
+
|
22
|
+
port ||= 11_300
|
23
|
+
connection_string = "#{uri.host}:#{port}"
|
24
|
+
@beanstalk = Beanstalk::Pool.new([connection_string])
|
25
|
+
|
26
|
+
@beanstalk.watch(queue_name)
|
27
|
+
@message_uri = "beanstalk://#{uri.host}:#{port}/#{queue_name}"
|
28
|
+
rescue StandardError => e
|
29
|
+
puts "Error connecting to Beanstalk, Host string, #{connection_string}"
|
30
|
+
if e.message == 'Beanstalk::NotConnected'
|
31
|
+
puts '***Most likely, beanstalk is not running. Start beanstalk, and try running this again.\n' \
|
32
|
+
"***If you still get this error, check beanstalk is running at, #{connection_string}"
|
33
|
+
abort
|
34
|
+
end
|
35
|
+
|
36
|
+
puts e.message
|
37
|
+
puts e.backtrace
|
38
|
+
end
|
39
|
+
# rubocop:enable Metrics/MethodLength,Metrics/AbcSize
|
40
|
+
|
41
|
+
def look
|
42
|
+
job = @beanstalk.reserve @timeout
|
43
|
+
send(job.body, "#{@message_uri}/#{job.id}")
|
44
|
+
job_body = job.body
|
45
|
+
job.delete
|
46
|
+
job_body
|
47
|
+
rescue StandardError => e
|
48
|
+
return if e.message == 'TIMED_OUT'
|
49
|
+
|
50
|
+
raise e
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -9,6 +9,7 @@ module RServiceBus2
|
|
9
9
|
|
10
10
|
# rubocop:disable Metrics/ClassLength
|
11
11
|
class MonitorDir < Monitor
|
12
|
+
# rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
12
13
|
def input_dir(uri)
|
13
14
|
# Pass the path through the Dir object to check syntax on startup
|
14
15
|
return Dir.new(uri.path) if File.writable?(uri.path)
|
@@ -27,44 +28,48 @@ module RServiceBus2
|
|
27
28
|
"***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
28
29
|
abort
|
29
30
|
end
|
31
|
+
# rubocop:enable Metrics/MethodLength,Metrics/AbcSize
|
30
32
|
|
31
|
-
def
|
32
|
-
|
33
|
-
@input_filter = []
|
34
|
-
|
35
|
-
return if uri.query.nil?
|
33
|
+
def initialise_archive(parts)
|
34
|
+
return unless parts.key?('archive')
|
36
35
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
puts '***** Archive file name templating not yet supported.'
|
43
|
-
puts "***** Directory's only."
|
44
|
-
abort
|
45
|
-
end
|
46
|
-
@archivedir = archiveuri.path
|
36
|
+
archiveuri = URI.parse(parts['archive'][0])
|
37
|
+
unless File.directory?(archiveuri.path)
|
38
|
+
puts '***** Archive file name templating not yet supported.'
|
39
|
+
puts "***** Directory's only."
|
40
|
+
abort
|
47
41
|
end
|
42
|
+
@archivedir = archiveuri.path
|
43
|
+
end
|
48
44
|
|
49
|
-
|
50
|
-
|
45
|
+
def initialise_input_filter(parts)
|
51
46
|
if parts['input_filter'].count > 1
|
52
|
-
puts 'Too many input_filters specified.'
|
53
|
-
puts '*** ZIP, or GZ are the only valid input_filters.'
|
47
|
+
puts 'Too many input_filters specified.\n*** ZIP, or GZ are the only valid input_filters.'
|
54
48
|
abort
|
55
49
|
end
|
56
50
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
else
|
61
|
-
puts 'Invalid input_filter specified.'
|
62
|
-
puts '*** ZIP, or GZ are the only valid input_filters.'
|
51
|
+
unless %w[ZIP GZ TAR].include?(parts['input_filter'][0]).nil?
|
52
|
+
puts 'Invalid input_filter specified.\n' \
|
53
|
+
'*** ZIP, or GZ are the only valid input_filters.'
|
63
54
|
abort
|
64
55
|
end
|
56
|
+
|
65
57
|
@input_filter << parts['input_filter'][0]
|
66
58
|
end
|
67
59
|
|
60
|
+
def connect(uri)
|
61
|
+
@path = input_dir(uri).path
|
62
|
+
@input_filter = []
|
63
|
+
|
64
|
+
return if uri.query.nil?
|
65
|
+
|
66
|
+
parts = CGI.parse(uri.query)
|
67
|
+
@querystringparts = parts
|
68
|
+
initialise_archive(parts)
|
69
|
+
|
70
|
+
initialise_input_filter(parts) if parts.key?('input_filter')
|
71
|
+
end
|
72
|
+
|
68
73
|
def process_content(content)
|
69
74
|
content
|
70
75
|
end
|
@@ -86,13 +91,14 @@ module RServiceBus2
|
|
86
91
|
# rubocop:disable Metrics/MethodLength
|
87
92
|
def read_content_from_file(file_path)
|
88
93
|
content = ''
|
89
|
-
if @input_filter.length
|
90
|
-
|
94
|
+
if @input_filter.length.positive?
|
95
|
+
case @input_filter[0]
|
96
|
+
when 'ZIP'
|
91
97
|
content = read_content_from_zip_file(file_path)
|
92
|
-
|
98
|
+
when 'GZ'
|
93
99
|
content = read_content_from_gz_file(file_path)
|
94
|
-
|
95
|
-
|
100
|
+
else
|
101
|
+
raise "#{@input_filter[0]} reader not implemented."
|
96
102
|
end
|
97
103
|
|
98
104
|
else
|
@@ -101,13 +107,34 @@ module RServiceBus2
|
|
101
107
|
|
102
108
|
content
|
103
109
|
end
|
110
|
+
# rubocop:enable Metrics/MethodLength
|
111
|
+
|
112
|
+
def archive_file(file_path, content)
|
113
|
+
basename = File.basename(file_path)
|
114
|
+
new_file_path = "#{@archivedir}/#{basename}.#{Time.now.strftime('%Y%m%d%H%M%S%L')}.zip"
|
115
|
+
RServiceBus2.log "Writing to archive, #{new_file_path}"
|
116
|
+
|
117
|
+
Zip::ZipOutputStream.open(new_file_path) do |zos|
|
118
|
+
zos.put_next_entry(basename)
|
119
|
+
zos.puts content
|
120
|
+
end
|
121
|
+
end
|
104
122
|
|
105
123
|
def process_path(file_path)
|
124
|
+
if File.file?(file_path) != true
|
125
|
+
RServiceBus2.log "Skipping directory, #{file_path}"
|
126
|
+
return
|
127
|
+
end
|
128
|
+
|
129
|
+
RServiceBus2.log "Ready to process, #{file_path}"
|
106
130
|
content = read_content_from_file(file_path)
|
107
131
|
payload = process_content(content)
|
108
132
|
|
109
|
-
send(payload, URI.parse(
|
110
|
-
|
133
|
+
send(payload, URI.parse(CGI.escape("file://#{file_path}")))
|
134
|
+
|
135
|
+
archive_file(file_path, content) unless @archivedir.nil?
|
136
|
+
|
137
|
+
File.unlink(file_path)
|
111
138
|
end
|
112
139
|
|
113
140
|
def look
|
@@ -116,28 +143,11 @@ module RServiceBus2
|
|
116
143
|
|
117
144
|
file_list = Dir.glob("#{@path}/*")
|
118
145
|
file_list.each do |file_path|
|
119
|
-
|
120
|
-
RServiceBus2.log "Skipping directory, #{file_path}"
|
121
|
-
next
|
122
|
-
end
|
123
|
-
RServiceBus2.log "Ready to process, #{file_path}"
|
124
|
-
content = process_path(file_path)
|
125
|
-
|
126
|
-
unless @archivedir.nil?
|
127
|
-
basename = File.basename(file_path)
|
128
|
-
new_file_path = "#{@archivedir}/#{basename}.#{Time.now.strftime('%Y%m%d%H%M%S%L')}.zip"
|
129
|
-
RServiceBus2.log "Writing to archive, #{new_file_path}"
|
130
|
-
|
131
|
-
Zip::ZipOutputStream.open(new_file_path) do |zos|
|
132
|
-
zos.put_next_entry(basename)
|
133
|
-
zos.puts content
|
134
|
-
end
|
135
|
-
end
|
136
|
-
File.unlink(file_path)
|
146
|
+
process_path(file_path)
|
137
147
|
|
138
148
|
file_processed += 1
|
139
|
-
RServiceBus2.log "Processed #{file_processed} of #{file_list.length}."
|
140
|
-
|
149
|
+
RServiceBus2.log "Processed #{file_processed} of #{file_list.length}.\nAllow system tick #{self.class.name}"
|
150
|
+
|
141
151
|
break if file_processed >= max_files_processed
|
142
152
|
end
|
143
153
|
end
|
@@ -9,78 +9,54 @@ module RServiceBus2
|
|
9
9
|
class MonitorDirNotifier < Monitor
|
10
10
|
attr_reader :path, :processing_folder, :filter
|
11
11
|
|
12
|
-
def
|
13
|
-
|
14
|
-
|
15
|
-
open_folder uri.path
|
16
|
-
unless File.writable?(uri.path)
|
17
|
-
puts "***** Directory is not writable, #{uri.path}."
|
18
|
-
puts "***** Make the directory, #{uri.path}, writable and try again."
|
19
|
-
abort
|
20
|
-
end
|
21
|
-
rescue Errno::ENOENT
|
22
|
-
puts "***** Directory does not exist, #{uri.path}."
|
23
|
-
puts "***** Create the directory, #{uri.path}, and try again."
|
24
|
-
puts "***** eg, mkdir #{uri.path}"
|
25
|
-
abort
|
26
|
-
rescue Errno::ENOTDIR
|
27
|
-
puts "***** The specified path does not point to a directory,
|
28
|
-
#{uri.path}."
|
29
|
-
puts "***** Either repoint path to a directory, or remove, #{uri.path},
|
30
|
-
and create it as a directory."
|
31
|
-
puts "***** eg, rm #{uri.path} && mkdir #{uri.path}"
|
32
|
-
abort
|
33
|
-
end
|
12
|
+
def validate_directory(path, param_name)
|
13
|
+
open_folder path
|
14
|
+
return if File.writable?(path)
|
34
15
|
|
35
|
-
|
16
|
+
puts "***** #{param_name} is not writable, #{path}.\n
|
17
|
+
***** Make the directory, #{path}, writable and try again."
|
18
|
+
abort
|
19
|
+
rescue Errno::ENOENT
|
20
|
+
puts "***** #{param_name} does not exist, #{path}.\n" \
|
21
|
+
"***** Create the directory, #{path}, and try again.\n" \
|
22
|
+
"***** eg, mkdir #{path}"
|
23
|
+
abort
|
24
|
+
rescue Errno::ENOTDIR
|
25
|
+
puts "***** The specified path does not point to a directory, #{path}.
|
26
|
+
***** Either repoint path to a directory, or remove, #{path}, and create it as a directory.
|
27
|
+
***** eg, rm #{path} && mkdir #{path}"
|
28
|
+
abort
|
29
|
+
end
|
36
30
|
|
31
|
+
def validate_processing_directory(uri)
|
37
32
|
if uri.query.nil?
|
38
|
-
puts '***** Processing Directory is not specified.'
|
39
|
-
|
40
|
-
|
41
|
-
puts "***** eg, '/#{uri.path}?processing=*ProcessingDir*"
|
33
|
+
puts '***** Processing Directory is not specified.' \
|
34
|
+
'***** Specify the Processing Directory as a query string in the Path URI' \
|
35
|
+
"***** eg, '/#{uri.path}?processing=*ProcessingDir*'"
|
42
36
|
abort
|
43
|
-
|
44
|
-
parts = CGI.parse(uri.query)
|
37
|
+
end
|
45
38
|
|
46
|
-
|
47
|
-
|
48
|
-
begin
|
49
|
-
open_folder processing_uri.path
|
50
|
-
unless File.writable?(processing_uri.path)
|
51
|
-
puts "***** 1Processing Directory is not writable,
|
52
|
-
#{processing_uri.path}."
|
53
|
-
puts "***** Make the directory, #{processing_uri.path},
|
54
|
-
writable and try again."
|
55
|
-
abort
|
56
|
-
end
|
57
|
-
rescue Errno::ENOENT
|
58
|
-
puts "***** Processing Directory does not exist,
|
59
|
-
#{processing_uri.path}."
|
60
|
-
puts "***** Create the directory, #{processing_uri.path}, and try
|
61
|
-
again."
|
62
|
-
puts "***** eg, mkdir #{processing_uri.path}"
|
63
|
-
abort
|
64
|
-
rescue Errno::ENOTDIR
|
65
|
-
puts "***** Processing Directory does not point to a directory,
|
66
|
-
#{processing_uri.path}."
|
67
|
-
puts "***** Either repoint path to a directory, or remove,
|
68
|
-
#{processing_uri.path}, and create it as a directory."
|
69
|
-
puts "***** eg, rm #{processing_uri.path} && mkdir
|
70
|
-
#{processing_uri.path}"
|
71
|
-
abort
|
72
|
-
end
|
39
|
+
parts = CGI.parse(uri.query)
|
40
|
+
return unless parts.key? 'processing'
|
73
41
|
|
74
|
-
|
75
|
-
|
42
|
+
processing_uri = URI.parse parts['processing'][0]
|
43
|
+
validate_directory processing_uri.path, 'Processing Directory'
|
44
|
+
@processing_folder = processing_uri.path
|
45
|
+
end
|
76
46
|
|
77
|
-
|
78
|
-
|
79
|
-
|
47
|
+
def connect(uri)
|
48
|
+
# Pass the path through the Dir object to check syntax on startup
|
49
|
+
|
50
|
+
validate_directory uri, 'Directory'
|
51
|
+
@path = uri.path
|
52
|
+
validate_processing_directory(uri)
|
53
|
+
|
54
|
+
@filter = '*'
|
55
|
+
@filter = parts['filter'][0] if parts.key? 'filter'
|
80
56
|
end
|
81
57
|
|
82
58
|
def look
|
83
|
-
file_list =
|
59
|
+
file_list = files
|
84
60
|
file_list.each do |file_path|
|
85
61
|
new_path = move_file(file_path, @processing_folder)
|
86
62
|
send(nil, URI.parse("file://#{new_path}"))
|
@@ -97,8 +73,8 @@ module RServiceBus2
|
|
97
73
|
Pathname.new(dest).join(filename)
|
98
74
|
end
|
99
75
|
|
100
|
-
def
|
101
|
-
Dir.glob(Pathname.new(
|
76
|
+
def files
|
77
|
+
Dir.glob(Pathname.new(@path).join(@filter)).select { |f| File.file?(f) }
|
102
78
|
end
|
103
79
|
end
|
104
80
|
end
|
@@ -57,6 +57,9 @@ module RServiceBus2
|
|
57
57
|
when 'dirnotifier'
|
58
58
|
require 'rservicebus2/monitor/dirnotifier'
|
59
59
|
monitor = MonitorDirNotifier.new(@host, name, uri)
|
60
|
+
when 'beanstalk'
|
61
|
+
require 'rservicebus2/monitor/beanstalk'
|
62
|
+
monitor = MonitorBeanstalk.new(@host, name, uri)
|
60
63
|
else
|
61
64
|
abort("Scheme, #{uri.scheme}, not recognised when configuring
|
62
65
|
Monitor, #{key}=#{val}")
|
data/lib/rservicebus2/mq/aws.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aws-sdk-sqs'
|
2
4
|
require 'aws-sdk-sts'
|
3
5
|
require 'rservicebus2/mq'
|
@@ -6,75 +8,59 @@ module RServiceBus2
|
|
6
8
|
# Beanstalk client implementation.
|
7
9
|
class MQAWS < MQ
|
8
10
|
# Connect to the broker
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
@max_job_size = 4_194_304
|
13
|
-
@region = region
|
11
|
+
def connect(region, _port)
|
12
|
+
@max_job_size = 4_194_304
|
13
|
+
@region = region
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end
|
15
|
+
sts_client = Aws::STS::Client.new(region: region)
|
16
|
+
@caller_identity_account = sts_client.get_caller_identity.account
|
17
|
+
rescue StandardError => e
|
18
|
+
puts 'Error connecting to AWS'
|
19
|
+
puts "Host string, #{region}"
|
20
|
+
puts e.message
|
21
|
+
puts e.backtrace
|
22
|
+
abort
|
24
23
|
end
|
25
24
|
|
26
25
|
def subscribe(queuename)
|
27
26
|
# For example:
|
28
27
|
# 'https://sqs.us-east-1.amazonaws.com/111111111111/my-queue'
|
29
|
-
@queue_url =
|
30
|
-
@caller_identity_account + '/' + queuename
|
28
|
+
@queue_url = "https://sqs.#{@region}.amazonaws.com/#{@caller_identity_account}/#{queuename}"
|
31
29
|
@sqs_client = Aws::SQS::Client.new(region: @region)
|
32
30
|
end
|
33
31
|
|
34
32
|
# Get next msg from queue
|
35
33
|
def pop
|
36
|
-
|
37
|
-
response = @sqs_client.receive_message(
|
38
|
-
queue_url: @queue_url,
|
39
|
-
max_number_of_messages: 1
|
40
|
-
)
|
34
|
+
response = @sqs_client.receive_message(queue_url: @queue_url, max_number_of_messages: 1)
|
41
35
|
|
42
|
-
|
43
|
-
raise NoMsgToProcess
|
44
|
-
end
|
36
|
+
raise NoMsgToProcess if response.messages.count.zero?
|
45
37
|
|
46
|
-
|
47
|
-
|
48
|
-
end
|
49
|
-
rescue StandardError => e
|
50
|
-
raise e
|
38
|
+
response.messages.each do |message|
|
39
|
+
@job = message
|
51
40
|
end
|
41
|
+
rescue StandardError => e
|
42
|
+
raise e
|
43
|
+
ensure
|
52
44
|
@job.body
|
53
45
|
end
|
54
46
|
|
55
47
|
def return_to_queue
|
56
|
-
# @job.release if @job
|
57
48
|
@job = nil
|
58
49
|
end
|
59
50
|
|
60
51
|
def ack
|
61
|
-
@sqs_client.delete_message({
|
62
|
-
queue_url: @queue_url,
|
63
|
-
receipt_handle: @job.receipt_handle
|
64
|
-
})
|
52
|
+
@sqs_client.delete_message({ queue_url: @queue_url, receipt_handle: @job.receipt_handle })
|
65
53
|
@job = nil
|
66
54
|
end
|
67
55
|
|
68
56
|
def send(queue_name, msg)
|
69
57
|
if msg.length > @max_job_size
|
70
|
-
puts '***Attempting to send a msg which will not fit on queue.'
|
71
|
-
|
72
|
-
|
73
|
-
#{@max_job_size}"
|
58
|
+
puts '***Attempting to send a msg which will not fit on queue.' \
|
59
|
+
"***Msg size, #{msg.length}, max msg size, #{@max_job_size}."
|
60
|
+
raise JobTooBigError, "Msg size, #{msg.length}, max msg size, #{@max_job_size}"
|
74
61
|
end
|
75
62
|
|
76
|
-
queue_url =
|
77
|
-
@caller_identity_account + '/' + queue_name
|
63
|
+
queue_url = "https://sqs.#{@region}.amazonaws.com/#{@caller_identity_account}/#{queue_name}"
|
78
64
|
|
79
65
|
@sqs_client.send_message(
|
80
66
|
queue_url: queue_url,
|
@@ -16,19 +16,17 @@ module RServiceBus2
|
|
16
16
|
@beanstalk = Beanstalk::Pool.new([string])
|
17
17
|
@max_job_size = @beanstalk.stats['max-job-size']
|
18
18
|
if @max_job_size < 4_194_304
|
19
|
-
puts
|
20
|
-
max-job-size, #{@max_job_size.to_f / (1024 * 1024)}m"
|
21
|
-
puts '***WARNING: Set the job size with the -z switch, eg
|
22
|
-
/usr/local/bin/beanstalkd -z 4194304'
|
19
|
+
puts '***WARNING: Lowest recommended.max-job-size is 4m, current ' \
|
20
|
+
"max-job-size, #{@max_job_size.to_f / (1024 * 1024)}m"
|
21
|
+
puts '***WARNING: Set the job size with the -z switch, eg ' \
|
22
|
+
'/usr/local/bin/beanstalkd -z 4194304'
|
23
23
|
end
|
24
24
|
rescue StandardError => e
|
25
25
|
puts 'Error connecting to Beanstalk'
|
26
26
|
puts "Host string, #{string}"
|
27
27
|
if e.message == 'Beanstalk::NotConnected'
|
28
|
-
puts '***Most likely, beanstalk is not running. Start beanstalk,
|
29
|
-
|
30
|
-
puts "***If you still get this error, check beanstalk is running
|
31
|
-
at, #{string}"
|
28
|
+
puts '***Most likely, beanstalk is not running. Start beanstalk, and try running this again.'
|
29
|
+
puts "***If you still get this error, check beanstalk is running at, #{string}"
|
32
30
|
else
|
33
31
|
puts e.message
|
34
32
|
puts e.backtrace
|
data/lib/rservicebus2/mq/file.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'fileutils'
|
2
4
|
require 'rservicebus2/mq'
|
3
5
|
|
@@ -5,18 +7,10 @@ module RServiceBus2
|
|
5
7
|
# Beanstalk client implementation.
|
6
8
|
class MQFile < MQ
|
7
9
|
# Connect to the broker
|
8
|
-
|
9
|
-
|
10
|
+
|
10
11
|
def initialize(uri)
|
11
|
-
|
12
|
-
@uri = uri
|
13
|
-
else
|
14
|
-
puts 'uri must be a valid URI'
|
15
|
-
abort
|
16
|
-
end
|
12
|
+
super
|
17
13
|
|
18
|
-
FileUtils.mkdir_p(@uri.path)
|
19
|
-
@local_queue_name = RServiceBus2.get_value('APPNAME', 'RServiceBus')
|
20
14
|
FileUtils.mkdir_p("#{@uri.path}/#{@local_queue_name}")
|
21
15
|
@timeout = RServiceBus2.get_value('QUEUE_TIMEOUT', '5').to_i
|
22
16
|
end
|
@@ -28,24 +22,23 @@ module RServiceBus2
|
|
28
22
|
end
|
29
23
|
|
30
24
|
# Get next msg from queue
|
25
|
+
# rubocop:disable Metrics/MethodLength
|
31
26
|
def pop
|
32
27
|
time = @timeout
|
33
|
-
while time
|
28
|
+
while time.positive?
|
34
29
|
files = Dir.glob("#{@uri.path}/#{@local_queue_name}/*.msg")
|
35
|
-
if files.
|
36
|
-
@file_path = files[0]
|
30
|
+
if files.positive?
|
31
|
+
@file_path = files[0]
|
37
32
|
@body = IO.read(@file_path)
|
38
|
-
|
33
|
+
return @body
|
39
34
|
end
|
40
|
-
|
41
|
-
|
35
|
+
time -= 1
|
36
|
+
sleep(1)
|
42
37
|
end
|
43
|
-
|
44
|
-
raise NoMsgToProcess if files.length == 0
|
45
|
-
end
|
46
38
|
|
47
|
-
|
39
|
+
raise NoMsgToProcess if files.length.zero?
|
48
40
|
end
|
41
|
+
# rubocop:enable Metrics/MethodLength
|
49
42
|
|
50
43
|
def ack
|
51
44
|
FileUtils.rm @file_path
|
@@ -53,7 +46,7 @@ module RServiceBus2
|
|
53
46
|
|
54
47
|
def send(queue_name, msg)
|
55
48
|
FileUtils.mkdir_p("#{@uri.path}/#{queue_name}")
|
56
|
-
IO.write("#{@uri.path}/#{queue_name}/#{rand
|
49
|
+
IO.write("#{@uri.path}/#{queue_name}/#{rand}.msg", msg)
|
57
50
|
end
|
58
51
|
end
|
59
52
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'redis'
|
2
4
|
require 'rservicebus2/mq'
|
3
5
|
|
@@ -5,38 +7,31 @@ module RServiceBus2
|
|
5
7
|
# Redis client implementation.
|
6
8
|
class MQRedis < MQ
|
7
9
|
# Connect to the broker
|
8
|
-
# rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
9
10
|
def connect(host, port)
|
10
11
|
port ||= 6379
|
11
12
|
string = "#{host}:#{port}"
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
puts "Host string, #{string}"
|
20
|
-
abort()
|
21
|
-
end
|
14
|
+
@redis = Redis.new(host: host, port: port)
|
15
|
+
rescue StandardError => e
|
16
|
+
puts e.message
|
17
|
+
puts 'Error connecting to Redis for mq'
|
18
|
+
puts "Host string, #{string}"
|
19
|
+
abort
|
22
20
|
end
|
23
21
|
|
24
22
|
# Connect to the queue
|
25
|
-
def subscribe(
|
23
|
+
def subscribe(queuename)
|
26
24
|
@queuename = queuename
|
27
25
|
end
|
28
26
|
|
29
27
|
# Get next msg from queue
|
30
28
|
def pop
|
31
|
-
if @redis.llen(
|
29
|
+
if @redis.llen(@queuename).zero?
|
32
30
|
sleep @timeout
|
33
|
-
raise NoMsgToProcess
|
31
|
+
raise NoMsgToProcess
|
34
32
|
end
|
35
33
|
|
36
|
-
|
37
|
-
end
|
38
|
-
|
39
|
-
def returnToQueue
|
34
|
+
@redis.lindex @queuename, 0
|
40
35
|
end
|
41
36
|
|
42
37
|
# "Commit" queue
|
@@ -45,9 +40,8 @@ module RServiceBus2
|
|
45
40
|
end
|
46
41
|
|
47
42
|
# At least called in the Host rescue block, to ensure all network links are healthy
|
48
|
-
def send(
|
49
|
-
@redis.rpush
|
43
|
+
def send(queue_name, msg)
|
44
|
+
@redis.rpush queue_name, msg
|
50
45
|
end
|
51
|
-
|
52
46
|
end
|
53
47
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rservicebus2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Guy Irvine
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-05-
|
11
|
+
date: 2021-05-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: beanstalk-client
|
@@ -129,6 +129,7 @@ files:
|
|
129
129
|
- lib/rservicebus2/monitor.rb
|
130
130
|
- lib/rservicebus2/monitor/awss3.rb
|
131
131
|
- lib/rservicebus2/monitor/awssqs.rb
|
132
|
+
- lib/rservicebus2/monitor/beanstalk.rb
|
132
133
|
- lib/rservicebus2/monitor/dir.rb
|
133
134
|
- lib/rservicebus2/monitor/dirnotifier.rb
|
134
135
|
- lib/rservicebus2/monitor/message.rb
|