legion-transport 0.1.0 → 1.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +53 -80
- data/.rspec +1 -0
- data/.rubocop.yml +3 -9
- data/CHANGELOG.md +9 -0
- data/Gemfile +0 -1
- data/README.md +3 -0
- data/bitbucket-pipelines.yml +1 -1
- data/legion-transport.gemspec +19 -14
- data/lib/legion/transport.rb +12 -10
- data/lib/legion/transport/common.rb +23 -18
- data/lib/legion/transport/connection.rb +56 -12
- data/lib/legion/transport/consumer.rb +8 -1
- data/lib/legion/transport/exchange.rb +44 -6
- data/lib/legion/transport/exchanges/crypt.rb +11 -0
- data/lib/legion/transport/exchanges/node.rb +11 -0
- data/lib/legion/transport/message.rb +100 -2
- data/lib/legion/transport/messages/check_subtask.rb +25 -0
- data/lib/legion/transport/messages/dynamic.rb +25 -0
- data/lib/legion/transport/messages/lex_register.rb +11 -19
- data/lib/legion/transport/messages/node_health.rb +21 -0
- data/lib/legion/transport/messages/request_cluster_secret.rb +33 -0
- data/lib/legion/transport/messages/subtask.rb +38 -0
- data/lib/legion/transport/messages/task_log.rb +36 -0
- data/lib/legion/transport/messages/task_update.rb +3 -24
- data/lib/legion/transport/queue.rb +65 -6
- data/lib/legion/transport/queues/node.rb +15 -0
- data/lib/legion/transport/queues/node_crypt.rb +21 -0
- data/lib/legion/transport/queues/node_status.rb +1 -1
- data/lib/legion/transport/queues/task_log.rb +1 -1
- data/lib/legion/transport/queues/task_update.rb +1 -1
- data/lib/legion/transport/settings.rb +72 -0
- data/lib/legion/transport/version.rb +1 -1
- data/settings/transport.json +5 -1
- data/sonar-project.properties +12 -0
- metadata +93 -49
- data/lib/legion/transport/connections/bunny.rb +0 -53
- data/lib/legion/transport/connections/common.rb +0 -36
- data/lib/legion/transport/connections/marchhare.rb +0 -20
- data/lib/legion/transport/consumers/bunny.rb +0 -11
- data/lib/legion/transport/consumers/common.rb +0 -8
- data/lib/legion/transport/consumers/marchhare.rb +0 -11
- data/lib/legion/transport/exchanges/bunny.rb +0 -28
- data/lib/legion/transport/exchanges/common.rb +0 -32
- data/lib/legion/transport/exchanges/marchhare.rb +0 -17
- data/lib/legion/transport/messages/base.rb +0 -22
- data/lib/legion/transport/messages/node_status.rb +0 -49
- data/lib/legion/transport/messages/task_check_subtask.rb +0 -35
- data/lib/legion/transport/messages/task_subtask.rb +0 -45
- data/lib/legion/transport/queues/bunny.rb +0 -26
- data/lib/legion/transport/queues/common.rb +0 -45
- data/lib/legion/transport/queues/marchhare.rb +0 -26
@@ -1,6 +1,13 @@
|
|
1
1
|
module Legion
|
2
2
|
module Transport
|
3
|
-
class
|
3
|
+
class Consumer < Legion::Transport::CONNECTOR::Consumer
|
4
|
+
include Legion::Transport::Common
|
5
|
+
attr_reader :consumer_tag
|
6
|
+
|
7
|
+
def initialize(queue:, no_ack: false, exclusive: false, consumer_tag: generate_consumer_tag, **opts)
|
8
|
+
@consumer_tag = consumer_tag
|
9
|
+
super(channel, queue, consumer_tag, no_ack, exclusive, opts)
|
10
|
+
end
|
4
11
|
end
|
5
12
|
end
|
6
13
|
end
|
@@ -1,12 +1,50 @@
|
|
1
1
|
module Legion
|
2
2
|
module Transport
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
class Exchange < Legion::Transport::CONNECTOR::Exchange
|
4
|
+
include Legion::Transport::Common
|
5
|
+
|
6
|
+
def initialize(exchange = exchange_name, options = {})
|
7
|
+
@options = options
|
8
|
+
@type = options[:type] || default_type
|
9
|
+
super(channel, @type, exchange, options_builder(default_options, exchange_options, @options))
|
10
|
+
rescue ::Bunny::PreconditionFailed, ::Bunny::ChannelAlreadyClosed
|
11
|
+
raise unless @retries.nil?
|
12
|
+
|
13
|
+
@retries = 1
|
14
|
+
delete_exchange(exchange)
|
15
|
+
retry
|
6
16
|
end
|
7
|
-
|
8
|
-
|
9
|
-
|
17
|
+
|
18
|
+
def delete_exchange(exchange)
|
19
|
+
Legion::Logging.warn "Exchange:#{exchange} exists with wrong parameters, deleting and creating"
|
20
|
+
channel.exchange_delete(exchange)
|
21
|
+
end
|
22
|
+
|
23
|
+
def default_options
|
24
|
+
hash = {}
|
25
|
+
hash[:durable] = true
|
26
|
+
hash[:auto_delete] = false
|
27
|
+
hash[:arguments] = {}
|
28
|
+
hash
|
29
|
+
end
|
30
|
+
|
31
|
+
def exchange_name
|
32
|
+
self.class.ancestors.first.to_s.split('::')[2].downcase
|
33
|
+
end
|
34
|
+
|
35
|
+
def exchange_options
|
36
|
+
{}
|
37
|
+
end
|
38
|
+
|
39
|
+
def delete(options = {})
|
40
|
+
super(options)
|
41
|
+
true
|
42
|
+
rescue ::Bunny::PreconditionFailed
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
def default_type
|
47
|
+
'topic'
|
10
48
|
end
|
11
49
|
end
|
12
50
|
end
|
@@ -1,8 +1,106 @@
|
|
1
1
|
module Legion
|
2
2
|
module Transport
|
3
3
|
class Message
|
4
|
-
|
5
|
-
|
4
|
+
include Legion::Transport::Common
|
5
|
+
|
6
|
+
def initialize(**options)
|
7
|
+
@options = options
|
8
|
+
validate
|
9
|
+
end
|
10
|
+
|
11
|
+
def publish(options = @options) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
12
|
+
raise unless @valid
|
13
|
+
|
14
|
+
exchange_dest = exchange.respond_to?(:new) ? exchange.new : exchange
|
15
|
+
exchange_dest.publish(encode_message,
|
16
|
+
routing_key: routing_key || '',
|
17
|
+
content_type: options[:content_type] || content_type,
|
18
|
+
content_encoding: options[:content_encoding] || content_encoding,
|
19
|
+
type: options[:type] || type,
|
20
|
+
priority: options[:priority] || priority,
|
21
|
+
expiration: options[:expiration] || expiration,
|
22
|
+
headers: headers)
|
23
|
+
end
|
24
|
+
|
25
|
+
def message
|
26
|
+
@options
|
27
|
+
end
|
28
|
+
|
29
|
+
def encode_message
|
30
|
+
message_payload = message
|
31
|
+
message_payload = Legion::JSON.dump(message_payload) unless message_payload.is_a? String
|
32
|
+
|
33
|
+
if encrypt?
|
34
|
+
encrypted = Legion::Crypt.encrypt(message_payload)
|
35
|
+
headers[:iv] = encrypted[:iv]
|
36
|
+
@options[:content_encoding] = 'encrypted/cs'
|
37
|
+
return encrypted[:enciphered_message]
|
38
|
+
else
|
39
|
+
@options[:content_encoding] = 'identity'
|
40
|
+
end
|
41
|
+
|
42
|
+
message_payload
|
43
|
+
end
|
44
|
+
|
45
|
+
def encrypt_message(message, _type = 'cs')
|
46
|
+
Legion::Crypt.encrypt(message)
|
47
|
+
end
|
48
|
+
|
49
|
+
def encrypt?
|
50
|
+
Legion::Settings[:transport][:messages][:encrypt] && Legion::Settings[:crypt][:cs_encrypt_ready]
|
51
|
+
end
|
52
|
+
|
53
|
+
def exchange_name
|
54
|
+
lex = self.class.ancestors.first.to_s.split('::')[2].downcase
|
55
|
+
"Legion::Extensions::#{lex.capitalize}::Transport::Exchanges::#{lex.capitalize}"
|
56
|
+
end
|
57
|
+
|
58
|
+
def exchange
|
59
|
+
Kernel.const_get(exchange_name)
|
60
|
+
end
|
61
|
+
|
62
|
+
def headers
|
63
|
+
@options[:headers] ||= {}
|
64
|
+
%i[task_id relationship_id trigger_namespace_id trigger_function_id parent_id master_id runner_namespace runner_class namespace_id function_id function chain_id debug].each do |header| # rubocop:disable Layout/LineLength
|
65
|
+
next unless @options.key? header
|
66
|
+
|
67
|
+
@options[:headers][header] = @options[header].to_s
|
68
|
+
end
|
69
|
+
@options[:headers]
|
70
|
+
rescue StandardError => e
|
71
|
+
Legion::Logging.error e.message
|
72
|
+
Legion::Logging.error e.backtrace
|
73
|
+
end
|
74
|
+
|
75
|
+
def priority
|
76
|
+
0
|
77
|
+
end
|
78
|
+
|
79
|
+
def expiration
|
80
|
+
nil
|
81
|
+
end
|
82
|
+
|
83
|
+
def content_type
|
84
|
+
'application/json'
|
85
|
+
end
|
86
|
+
|
87
|
+
def content_encoding
|
88
|
+
'identity'
|
89
|
+
end
|
90
|
+
|
91
|
+
def type
|
92
|
+
'task'
|
93
|
+
end
|
94
|
+
|
95
|
+
def timestamp
|
96
|
+
now.to_i
|
97
|
+
end
|
98
|
+
|
99
|
+
def validate
|
100
|
+
@valid = true
|
101
|
+
end
|
6
102
|
end
|
7
103
|
end
|
8
104
|
end
|
105
|
+
|
106
|
+
Dir["#{__dir__}/messages/*.rb"].sort.each { |file| require file }
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class CheckSubtask < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Task
|
9
|
+
end
|
10
|
+
|
11
|
+
def exchange_name
|
12
|
+
'Legion::Transport::Exchanges::Task'
|
13
|
+
end
|
14
|
+
|
15
|
+
def routing_key
|
16
|
+
'task.subtask.check'
|
17
|
+
end
|
18
|
+
|
19
|
+
def validate
|
20
|
+
@valid = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Legion::Transport::Messages # rubocop:disable Style/ClassAndModuleChildren
|
2
|
+
class Dynamic < Legion::Transport::Message
|
3
|
+
attr_accessor :options
|
4
|
+
|
5
|
+
def type
|
6
|
+
'task'
|
7
|
+
end
|
8
|
+
|
9
|
+
def message
|
10
|
+
{ args: @options[:args] || @options }
|
11
|
+
end
|
12
|
+
|
13
|
+
def routing_key
|
14
|
+
"#{function.runner.extension.values[:name]}.#{function.runner.values[:name]}.#{function.values[:name]}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def exchange
|
18
|
+
Legion::Transport::Exchange.new(function.runner.extension.values[:exchange])
|
19
|
+
end
|
20
|
+
|
21
|
+
def function
|
22
|
+
@function ||= Legion::Data::Model::Function[@options[:function_id]]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -4,32 +4,24 @@ module Legion
|
|
4
4
|
module Transport
|
5
5
|
module Messages
|
6
6
|
class LexRegister < Legion::Transport::Message
|
7
|
-
def initialize(namespace, method, options = {})
|
8
|
-
@namespace = namespace
|
9
|
-
@method = method
|
10
|
-
@options = options
|
11
|
-
end
|
12
|
-
|
13
7
|
def exchange
|
14
8
|
Legion::Transport::Exchanges::Lex
|
15
9
|
end
|
16
10
|
|
17
11
|
def routing_key
|
18
|
-
'lex.
|
12
|
+
'lex.register.save'
|
19
13
|
end
|
20
14
|
|
21
|
-
def
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
# raise unless method.is_a? String
|
32
|
-
# raise unless options.is_a? Hash
|
15
|
+
def validate
|
16
|
+
unless @options[:runner_namespace].is_a? String
|
17
|
+
# raise "runner_namespace is a #{@options[:runner_namespace].class}"
|
18
|
+
end
|
19
|
+
unless @options[:extension_namespace].is_a? String
|
20
|
+
# raise "extension_namespace is a #{@options[:extension_namespace].class}"
|
21
|
+
end
|
22
|
+
unless @options[:function].is_a?(String) || @options[:function].is_a?(Symbol)
|
23
|
+
# raise "function is a #{@options[:function].class}"
|
24
|
+
end
|
33
25
|
|
34
26
|
@valid = true
|
35
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'legion/transport/exchanges/lex'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class NodeHealth < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Node
|
9
|
+
end
|
10
|
+
|
11
|
+
def routing_key
|
12
|
+
'health'
|
13
|
+
end
|
14
|
+
|
15
|
+
def validate
|
16
|
+
@valid = true
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Legion::Transport::Messages # rubocop:disable Style/ClassAndModuleChildren
|
2
|
+
class RequestClusterSecret < Legion::Transport::Message
|
3
|
+
def routing_key
|
4
|
+
'node.crypt.push_cluster_secret'
|
5
|
+
end
|
6
|
+
|
7
|
+
def message
|
8
|
+
{ function: 'push_cluster_secret',
|
9
|
+
node_name: Legion::Settings[:client][:name],
|
10
|
+
queue_name: "node.#{Legion::Settings[:client][:name]}",
|
11
|
+
runner_class: 'Legion::Extensions::Node::Runners::Crypt',
|
12
|
+
# public_key: Base64.encode64(Legion::Crypt.public_key) }
|
13
|
+
public_key: Legion::Crypt.public_key }
|
14
|
+
end
|
15
|
+
|
16
|
+
def exchange
|
17
|
+
require 'legion/transport/exchanges/node'
|
18
|
+
Legion::Transport::Exchanges::Node
|
19
|
+
end
|
20
|
+
|
21
|
+
def encrypt?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def type
|
26
|
+
'task'
|
27
|
+
end
|
28
|
+
|
29
|
+
def validate
|
30
|
+
@valid = true
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class SubTask < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Task
|
9
|
+
end
|
10
|
+
|
11
|
+
def message
|
12
|
+
{
|
13
|
+
transformation: @options[:transformation] || '{}',
|
14
|
+
conditions: @options[:conditions] || '{}',
|
15
|
+
results: @options[:results] || '{}'
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
def routing_key # rubocop:disable Metrics/AbcSize
|
20
|
+
if @options[:conditions].is_a?(String) && @options[:conditions].length > 2
|
21
|
+
'task.subtask.conditioner'
|
22
|
+
elsif @options[:transformation].is_a?(String) && @options[:transformation].length > 2
|
23
|
+
'task.subtask.transform'
|
24
|
+
elsif @options[:function_id].is_a? Integer
|
25
|
+
function = Legion::Data::Model::Function[@options[:function_id]]
|
26
|
+
"#{function.runner.extension.values[:exchange]}.#{function.runner.values[:queue]}.#{function.values[:name]}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def validate
|
31
|
+
raise TypeError unless @options[:function].is_a? String
|
32
|
+
|
33
|
+
@valid = true
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class TaskLog < Legion::Transport::Message
|
7
|
+
def routing_key
|
8
|
+
"task.logs.create.#{@options[:task_id]}"
|
9
|
+
end
|
10
|
+
|
11
|
+
def exchange
|
12
|
+
Legion::Transport::Exchanges::Task
|
13
|
+
end
|
14
|
+
|
15
|
+
def message
|
16
|
+
@options[:function] = 'add_log'
|
17
|
+
@options[:runner_class] = 'Legion::Extensions::Tasker::Runners::Log'
|
18
|
+
@options
|
19
|
+
end
|
20
|
+
|
21
|
+
def generate_task?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate
|
26
|
+
@options[:task_id] = @options[:task_id].to_i if @options[:task_id].is_a? String
|
27
|
+
unless @options[:task_id].is_a? Integer
|
28
|
+
raise "task_id must be an integer but is #{@options[:task_id].class}(#{@options[:task_id]})"
|
29
|
+
end
|
30
|
+
|
31
|
+
@valid = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -11,15 +11,6 @@ module Legion
|
|
11
11
|
module Transport
|
12
12
|
module Messages
|
13
13
|
class TaskUpdate < Legion::Transport::Message
|
14
|
-
attr_accessor :status
|
15
|
-
attr_reader :task_id, :valid
|
16
|
-
def initialize(task_id, status = 'unknown', _options = {})
|
17
|
-
@status = status
|
18
|
-
@task_id = task_id
|
19
|
-
@routing_key = 'task.update'
|
20
|
-
validate
|
21
|
-
end
|
22
|
-
|
23
14
|
def routing_key
|
24
15
|
'task.update'
|
25
16
|
end
|
@@ -28,23 +19,11 @@ module Legion
|
|
28
19
|
Legion::Transport::Exchanges::Task
|
29
20
|
end
|
30
21
|
|
31
|
-
def message(status = @status, task_id = @task_id, options = {})
|
32
|
-
Legion::JSON.dump(args: { status: status, task_id: task_id, options: options })
|
33
|
-
end
|
34
|
-
|
35
|
-
def validate(status = @status, task_id = @task_id)
|
36
|
-
raise Legion::Exception::InvalidTaskId unless task_id.is_a? Integer
|
37
|
-
raise Legion::Exception::InvalidTaskStatus unless valid_status.include? status
|
38
|
-
|
39
|
-
@valid = true
|
40
|
-
end
|
41
|
-
|
42
22
|
def valid_status
|
43
23
|
conditioner = ['conditioner.queued', 'conditioner.failed', 'conditioner.exception']
|
44
|
-
|
45
|
-
task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception']
|
46
|
-
|
47
|
-
status
|
24
|
+
transformer = ['transformer.queued', 'transformer.succeeded', 'transformer.exception']
|
25
|
+
task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception', 'task.delayed']
|
26
|
+
conditioner + transformer + task
|
48
27
|
end
|
49
28
|
end
|
50
29
|
end
|