legion-transport-java 1.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +89 -0
- data/.gitignore +13 -0
- data/.rspec +1 -0
- data/.rubocop.yml +21 -0
- data/CHANGELOG.md +9 -0
- data/Gemfile +3 -0
- data/README.md +3 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/bitbucket-pipelines.yml +24 -0
- data/legion-transport.gemspec +43 -0
- data/lib/legion/transport.rb +21 -0
- data/lib/legion/transport/common.rb +76 -0
- data/lib/legion/transport/connection.rb +62 -0
- data/lib/legion/transport/consumer.rb +13 -0
- data/lib/legion/transport/exchange.rb +51 -0
- data/lib/legion/transport/exchanges/crypt.rb +11 -0
- data/lib/legion/transport/exchanges/lex.rb +11 -0
- data/lib/legion/transport/exchanges/node.rb +11 -0
- data/lib/legion/transport/exchanges/task.rb +11 -0
- data/lib/legion/transport/message.rb +106 -0
- data/lib/legion/transport/messages/check_subtask.rb +25 -0
- data/lib/legion/transport/messages/dynamic.rb +25 -0
- data/lib/legion/transport/messages/lex_register.rb +31 -0
- data/lib/legion/transport/messages/node_health.rb +21 -0
- data/lib/legion/transport/messages/request_cluster_secret.rb +33 -0
- data/lib/legion/transport/messages/subtask.rb +38 -0
- data/lib/legion/transport/messages/task_log.rb +36 -0
- data/lib/legion/transport/messages/task_update.rb +31 -0
- data/lib/legion/transport/queue.rb +72 -0
- data/lib/legion/transport/queues/node.rb +15 -0
- data/lib/legion/transport/queues/node_crypt.rb +21 -0
- data/lib/legion/transport/queues/node_status.rb +21 -0
- data/lib/legion/transport/queues/task_log.rb +21 -0
- data/lib/legion/transport/queues/task_update.rb +21 -0
- data/lib/legion/transport/settings.rb +72 -0
- data/lib/legion/transport/version.rb +5 -0
- data/settings/transport.json +5 -0
- data/sonar-project.properties +12 -0
- metadata +269 -0
@@ -0,0 +1,106 @@
|
|
1
|
+
module Legion
|
2
|
+
module Transport
|
3
|
+
class Message
|
4
|
+
include Legion::Transport::Common
|
5
|
+
|
6
|
+
def initialize(**options)
|
7
|
+
@options = options
|
8
|
+
validate
|
9
|
+
end
|
10
|
+
|
11
|
+
def publish(options = @options) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
12
|
+
raise unless @valid
|
13
|
+
|
14
|
+
exchange_dest = exchange.respond_to?(:new) ? exchange.new : exchange
|
15
|
+
exchange_dest.publish(encode_message,
|
16
|
+
routing_key: routing_key || '',
|
17
|
+
content_type: options[:content_type] || content_type,
|
18
|
+
content_encoding: options[:content_encoding] || content_encoding,
|
19
|
+
type: options[:type] || type,
|
20
|
+
priority: options[:priority] || priority,
|
21
|
+
expiration: options[:expiration] || expiration,
|
22
|
+
headers: headers)
|
23
|
+
end
|
24
|
+
|
25
|
+
def message
|
26
|
+
@options
|
27
|
+
end
|
28
|
+
|
29
|
+
def encode_message
|
30
|
+
message_payload = message
|
31
|
+
message_payload = Legion::JSON.dump(message_payload) unless message_payload.is_a? String
|
32
|
+
|
33
|
+
if encrypt?
|
34
|
+
encrypted = Legion::Crypt.encrypt(message_payload)
|
35
|
+
headers[:iv] = encrypted[:iv]
|
36
|
+
@options[:content_encoding] = 'encrypted/cs'
|
37
|
+
return encrypted[:enciphered_message]
|
38
|
+
else
|
39
|
+
@options[:content_encoding] = 'identity'
|
40
|
+
end
|
41
|
+
|
42
|
+
message_payload
|
43
|
+
end
|
44
|
+
|
45
|
+
def encrypt_message(message, _type = 'cs')
|
46
|
+
Legion::Crypt.encrypt(message)
|
47
|
+
end
|
48
|
+
|
49
|
+
def encrypt?
|
50
|
+
Legion::Settings[:transport][:messages][:encrypt] && Legion::Settings[:crypt][:cs_encrypt_ready]
|
51
|
+
end
|
52
|
+
|
53
|
+
def exchange_name
|
54
|
+
lex = self.class.ancestors.first.to_s.split('::')[2].downcase
|
55
|
+
"Legion::Extensions::#{lex.capitalize}::Transport::Exchanges::#{lex.capitalize}"
|
56
|
+
end
|
57
|
+
|
58
|
+
def exchange
|
59
|
+
Kernel.const_get(exchange_name)
|
60
|
+
end
|
61
|
+
|
62
|
+
def headers
|
63
|
+
@options[:headers] ||= {}
|
64
|
+
%i[task_id relationship_id trigger_namespace_id trigger_function_id parent_id master_id runner_namespace runner_class namespace_id function_id function chain_id debug].each do |header| # rubocop:disable Layout/LineLength
|
65
|
+
next unless @options.key? header
|
66
|
+
|
67
|
+
@options[:headers][header] = @options[header].to_s
|
68
|
+
end
|
69
|
+
@options[:headers]
|
70
|
+
rescue StandardError => e
|
71
|
+
Legion::Logging.error e.message
|
72
|
+
Legion::Logging.error e.backtrace
|
73
|
+
end
|
74
|
+
|
75
|
+
def priority
|
76
|
+
0
|
77
|
+
end
|
78
|
+
|
79
|
+
def expiration
|
80
|
+
nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def content_type
|
84
|
+
'application/json'
|
85
|
+
end
|
86
|
+
|
87
|
+
def content_encoding
|
88
|
+
'identity'
|
89
|
+
end
|
90
|
+
|
91
|
+
def type
|
92
|
+
'task'
|
93
|
+
end
|
94
|
+
|
95
|
+
def timestamp
|
96
|
+
now.to_i
|
97
|
+
end
|
98
|
+
|
99
|
+
def validate
|
100
|
+
@valid = true
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
Dir["#{__dir__}/messages/*.rb"].sort.each { |file| require file }
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class CheckSubtask < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Task
|
9
|
+
end
|
10
|
+
|
11
|
+
def exchange_name
|
12
|
+
'Legion::Transport::Exchanges::Task'
|
13
|
+
end
|
14
|
+
|
15
|
+
def routing_key
|
16
|
+
'task.subtask.check'
|
17
|
+
end
|
18
|
+
|
19
|
+
def validate
|
20
|
+
@valid = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Legion::Transport::Messages # rubocop:disable Style/ClassAndModuleChildren
|
2
|
+
class Dynamic < Legion::Transport::Message
|
3
|
+
attr_accessor :options
|
4
|
+
|
5
|
+
def type
|
6
|
+
'task'
|
7
|
+
end
|
8
|
+
|
9
|
+
def message
|
10
|
+
{ args: @options[:args] || @options }
|
11
|
+
end
|
12
|
+
|
13
|
+
def routing_key
|
14
|
+
"#{function.runner.extension.values[:name]}.#{function.runner.values[:name]}.#{function.values[:name]}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def exchange
|
18
|
+
Legion::Transport::Exchange.new(function.runner.extension.values[:exchange])
|
19
|
+
end
|
20
|
+
|
21
|
+
def function
|
22
|
+
@function ||= Legion::Data::Model::Function[@options[:function_id]]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'legion/transport/exchanges/lex'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class LexRegister < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Lex
|
9
|
+
end
|
10
|
+
|
11
|
+
def routing_key
|
12
|
+
'lex.register.save'
|
13
|
+
end
|
14
|
+
|
15
|
+
def validate
|
16
|
+
unless @options[:runner_namespace].is_a? String
|
17
|
+
# raise "runner_namespace is a #{@options[:runner_namespace].class}"
|
18
|
+
end
|
19
|
+
unless @options[:extension_namespace].is_a? String
|
20
|
+
# raise "extension_namespace is a #{@options[:extension_namespace].class}"
|
21
|
+
end
|
22
|
+
unless @options[:function].is_a?(String) || @options[:function].is_a?(Symbol)
|
23
|
+
# raise "function is a #{@options[:function].class}"
|
24
|
+
end
|
25
|
+
|
26
|
+
@valid = true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'legion/transport/exchanges/lex'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class NodeHealth < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Node
|
9
|
+
end
|
10
|
+
|
11
|
+
def routing_key
|
12
|
+
'health'
|
13
|
+
end
|
14
|
+
|
15
|
+
def validate
|
16
|
+
@valid = true
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Legion::Transport::Messages # rubocop:disable Style/ClassAndModuleChildren
|
2
|
+
class RequestClusterSecret < Legion::Transport::Message
|
3
|
+
def routing_key
|
4
|
+
'node.crypt.push_cluster_secret'
|
5
|
+
end
|
6
|
+
|
7
|
+
def message
|
8
|
+
{ function: 'push_cluster_secret',
|
9
|
+
node_name: Legion::Settings[:client][:name],
|
10
|
+
queue_name: "node.#{Legion::Settings[:client][:name]}",
|
11
|
+
runner_class: 'Legion::Extensions::Node::Runners::Crypt',
|
12
|
+
# public_key: Base64.encode64(Legion::Crypt.public_key) }
|
13
|
+
public_key: Legion::Crypt.public_key }
|
14
|
+
end
|
15
|
+
|
16
|
+
def exchange
|
17
|
+
require 'legion/transport/exchanges/node'
|
18
|
+
Legion::Transport::Exchanges::Node
|
19
|
+
end
|
20
|
+
|
21
|
+
def encrypt?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def type
|
26
|
+
'task'
|
27
|
+
end
|
28
|
+
|
29
|
+
def validate
|
30
|
+
@valid = true
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class SubTask < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Task
|
9
|
+
end
|
10
|
+
|
11
|
+
def message
|
12
|
+
{
|
13
|
+
transformation: @options[:transformation] || '{}',
|
14
|
+
conditions: @options[:conditions] || '{}',
|
15
|
+
results: @options[:results] || '{}'
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
def routing_key # rubocop:disable Metrics/AbcSize
|
20
|
+
if @options[:conditions].is_a?(String) && @options[:conditions].length > 2
|
21
|
+
'task.subtask.conditioner'
|
22
|
+
elsif @options[:transformation].is_a?(String) && @options[:transformation].length > 2
|
23
|
+
'task.subtask.transform'
|
24
|
+
elsif @options[:function_id].is_a? Integer
|
25
|
+
function = Legion::Data::Model::Function[@options[:function_id]]
|
26
|
+
"#{function.runner.extension.values[:exchange]}.#{function.runner.values[:queue]}.#{function.values[:name]}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def validate
|
31
|
+
raise TypeError unless @options[:function].is_a? String
|
32
|
+
|
33
|
+
@valid = true
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class TaskLog < Legion::Transport::Message
|
7
|
+
def routing_key
|
8
|
+
"task.logs.create.#{@options[:task_id]}"
|
9
|
+
end
|
10
|
+
|
11
|
+
def exchange
|
12
|
+
Legion::Transport::Exchanges::Task
|
13
|
+
end
|
14
|
+
|
15
|
+
def message
|
16
|
+
@options[:function] = 'add_log'
|
17
|
+
@options[:runner_class] = 'Legion::Extensions::Tasker::Runners::Log'
|
18
|
+
@options
|
19
|
+
end
|
20
|
+
|
21
|
+
def generate_task?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate
|
26
|
+
@options[:task_id] = @options[:task_id].to_i if @options[:task_id].is_a? String
|
27
|
+
unless @options[:task_id].is_a? Integer
|
28
|
+
raise "task_id must be an integer but is #{@options[:task_id].class}(#{@options[:task_id]})"
|
29
|
+
end
|
30
|
+
|
31
|
+
@valid = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Exception
|
5
|
+
class InvalidTaskStatus; end
|
6
|
+
class InvalidTaskId; end
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
module Legion
|
11
|
+
module Transport
|
12
|
+
module Messages
|
13
|
+
class TaskUpdate < Legion::Transport::Message
|
14
|
+
def routing_key
|
15
|
+
'task.update'
|
16
|
+
end
|
17
|
+
|
18
|
+
def exchange
|
19
|
+
Legion::Transport::Exchanges::Task
|
20
|
+
end
|
21
|
+
|
22
|
+
def valid_status
|
23
|
+
conditioner = ['conditioner.queued', 'conditioner.failed', 'conditioner.exception']
|
24
|
+
transformer = ['transformer.queued', 'transformer.succeeded', 'transformer.exception']
|
25
|
+
task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception', 'task.delayed']
|
26
|
+
conditioner + transformer + task
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module Legion
|
2
|
+
module Transport
|
3
|
+
class Queue < Legion::Transport::CONNECTOR::Queue
|
4
|
+
include Legion::Transport::Common
|
5
|
+
|
6
|
+
def initialize(queue = queue_name, options = {})
|
7
|
+
retries ||= 0
|
8
|
+
@options = options
|
9
|
+
super(channel, queue, options_builder(default_options, queue_options, options))
|
10
|
+
rescue ::Bunny::PreconditionFailed
|
11
|
+
retries.zero? ? retries = 1 : raise
|
12
|
+
recreate_queue(channel, queue)
|
13
|
+
retry
|
14
|
+
end
|
15
|
+
|
16
|
+
def recreate_queue(channel, queue)
|
17
|
+
Legion::Logging.warn "Queue:#{queue} exists with wrong parameters, deleting and creating"
|
18
|
+
queue = ::Bunny::Queue.new(channel, queue, no_declare: true, passive: true)
|
19
|
+
queue.delete(if_empty: true)
|
20
|
+
end
|
21
|
+
|
22
|
+
def default_options
|
23
|
+
hash = {}
|
24
|
+
hash[:manual_ack] = true
|
25
|
+
hash[:durable] = true
|
26
|
+
hash[:exclusive] = false
|
27
|
+
hash[:block] = false
|
28
|
+
hash[:auto_delete] = false
|
29
|
+
hash[:arguments] = {
|
30
|
+
'x-max-priority': 255,
|
31
|
+
'x-overflow': 'reject-publish',
|
32
|
+
'x-dead-letter-exchange': "#{self.class.ancestors.first.to_s.split('::')[2].downcase}.dlx"
|
33
|
+
}
|
34
|
+
hash
|
35
|
+
end
|
36
|
+
|
37
|
+
def queue_options
|
38
|
+
{}
|
39
|
+
end
|
40
|
+
|
41
|
+
def queue_name
|
42
|
+
ancestor = self.class.ancestors.first.to_s.split('::')
|
43
|
+
name = if ancestor[5].scan(/[A-Z]/).length > 1
|
44
|
+
ancestor[5].gsub!(/(.)([A-Z])/, '\1_\2').downcase!
|
45
|
+
else
|
46
|
+
ancestor[5].downcase!
|
47
|
+
end
|
48
|
+
"#{ancestor[2].downcase}.#{name}"
|
49
|
+
end
|
50
|
+
|
51
|
+
def delete(options = { if_unused: true, if_empty: true })
|
52
|
+
super(options)
|
53
|
+
true
|
54
|
+
rescue ::Bunny::PreconditionFailed
|
55
|
+
false
|
56
|
+
end
|
57
|
+
|
58
|
+
def acknowledge(delivery_tag)
|
59
|
+
channel.acknowledge(delivery_tag)
|
60
|
+
end
|
61
|
+
|
62
|
+
def reject(delivery_tag, requeue: false)
|
63
|
+
channel.reject(delivery_tag, requeue)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
require_relative 'queues/node'
|
70
|
+
require_relative 'queues/node_status'
|
71
|
+
require_relative 'queues/task_log'
|
72
|
+
require_relative 'queues/task_update'
|