legion-transport 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +12 -12
- data/.idea/.rakeTasks +7 -0
- data/.idea/legion-transport.iml +45 -0
- data/.idea/misc.xml +7 -0
- data/.idea/modules.xml +8 -0
- data/.idea/vagrant.xml +7 -0
- data/.idea/workspace.xml +14 -0
- data/.rspec +1 -0
- data/.rubocop.yml +1 -8
- data/CHANGELOG.md +0 -0
- data/Gemfile +0 -1
- data/README.md +35 -0
- data/legion-transport.gemspec +15 -5
- data/lib/legion/transport.rb +11 -9
- data/lib/legion/transport/common.rb +23 -18
- data/lib/legion/transport/connection.rb +55 -12
- data/lib/legion/transport/consumer.rb +7 -1
- data/lib/legion/transport/exchange.rb +44 -6
- data/lib/legion/transport/exchanges/crypt.rb +11 -0
- data/lib/legion/transport/exchanges/node.rb +11 -0
- data/lib/legion/transport/message.rb +94 -2
- data/lib/legion/transport/messages/check_subtask.rb +25 -0
- data/lib/legion/transport/messages/dynamic.rb +24 -0
- data/lib/legion/transport/messages/lex_register.rb +11 -19
- data/lib/legion/transport/messages/node_health.rb +21 -0
- data/lib/legion/transport/messages/request_cluster_secret.rb +32 -0
- data/lib/legion/transport/messages/subtask.rb +38 -0
- data/lib/legion/transport/messages/task_log.rb +36 -0
- data/lib/legion/transport/messages/task_update.rb +3 -23
- data/lib/legion/transport/queue.rb +63 -6
- data/lib/legion/transport/queues/node.rb +15 -0
- data/lib/legion/transport/queues/node_status.rb +1 -1
- data/lib/legion/transport/queues/task_log.rb +1 -1
- data/lib/legion/transport/queues/task_update.rb +1 -1
- data/lib/legion/transport/settings.rb +64 -0
- data/lib/legion/transport/version.rb +1 -1
- data/settings/transport.json +9 -1
- metadata +81 -33
- data/lib/legion/transport/connections/bunny.rb +0 -53
- data/lib/legion/transport/connections/common.rb +0 -36
- data/lib/legion/transport/connections/marchhare.rb +0 -20
- data/lib/legion/transport/consumers/bunny.rb +0 -11
- data/lib/legion/transport/consumers/common.rb +0 -8
- data/lib/legion/transport/consumers/marchhare.rb +0 -11
- data/lib/legion/transport/exchanges/bunny.rb +0 -28
- data/lib/legion/transport/exchanges/common.rb +0 -32
- data/lib/legion/transport/exchanges/marchhare.rb +0 -17
- data/lib/legion/transport/messages/base.rb +0 -22
- data/lib/legion/transport/messages/node_status.rb +0 -49
- data/lib/legion/transport/messages/task_check_subtask.rb +0 -35
- data/lib/legion/transport/messages/task_subtask.rb +0 -45
- data/lib/legion/transport/queues/bunny.rb +0 -26
- data/lib/legion/transport/queues/common.rb +0 -45
- data/lib/legion/transport/queues/marchhare.rb +0 -26
@@ -1,6 +1,12 @@
|
|
1
1
|
module Legion
|
2
2
|
module Transport
|
3
|
-
class
|
3
|
+
class Consumer < Legion::Transport::CONNECTOR::Consumer
|
4
|
+
include Legion::Transport::Common
|
5
|
+
attr_reader :consumer_tag
|
6
|
+
def initialize(queue:, no_ack: false, exclusive: false, consumer_tag: generate_consumer_tag, **opts)
|
7
|
+
@consumer_tag = consumer_tag
|
8
|
+
super(channel, queue, consumer_tag, no_ack, exclusive, opts)
|
9
|
+
end
|
4
10
|
end
|
5
11
|
end
|
6
12
|
end
|
@@ -1,12 +1,50 @@
|
|
1
1
|
module Legion
|
2
2
|
module Transport
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
class Exchange < Legion::Transport::CONNECTOR::Exchange
|
4
|
+
include Legion::Transport::Common
|
5
|
+
|
6
|
+
def initialize(exchange = exchange_name, options = {})
|
7
|
+
@options = options
|
8
|
+
@type = options[:type] || default_type
|
9
|
+
super(channel, @type, exchange, options_builder(default_options, exchange_options, @options))
|
10
|
+
rescue ::Bunny::PreconditionFailed, ::Bunny::ChannelAlreadyClosed
|
11
|
+
raise unless @retries.nil?
|
12
|
+
|
13
|
+
@retries = 1
|
14
|
+
delete_exchange(exchange)
|
15
|
+
retry
|
6
16
|
end
|
7
|
-
|
8
|
-
|
9
|
-
|
17
|
+
|
18
|
+
def delete_exchange(exchange)
|
19
|
+
Legion::Logging.warn "Exchange:#{exchange} exists with wrong parameters, deleting and creating"
|
20
|
+
channel.exchange_delete(exchange)
|
21
|
+
end
|
22
|
+
|
23
|
+
def default_options
|
24
|
+
hash = {}
|
25
|
+
hash[:durable] = true
|
26
|
+
hash[:auto_delete] = false
|
27
|
+
hash[:arguments] = {}
|
28
|
+
hash
|
29
|
+
end
|
30
|
+
|
31
|
+
def exchange_name
|
32
|
+
self.class.ancestors.first.to_s.split('::')[2].downcase
|
33
|
+
end
|
34
|
+
|
35
|
+
def exchange_options
|
36
|
+
{}
|
37
|
+
end
|
38
|
+
|
39
|
+
def delete(options = {})
|
40
|
+
super(options)
|
41
|
+
true
|
42
|
+
rescue ::Bunny::PreconditionFailed
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
def default_type
|
47
|
+
'topic'
|
10
48
|
end
|
11
49
|
end
|
12
50
|
end
|
@@ -1,8 +1,100 @@
|
|
1
1
|
module Legion
|
2
2
|
module Transport
|
3
3
|
class Message
|
4
|
-
|
5
|
-
|
4
|
+
include Legion::Transport::Common
|
5
|
+
|
6
|
+
def initialize(**options)
|
7
|
+
@options = options
|
8
|
+
validate
|
9
|
+
end
|
10
|
+
|
11
|
+
def publish(options = @options)
|
12
|
+
raise unless @valid
|
13
|
+
|
14
|
+
exchange_dest = exchange.respond_to?(:new) ? exchange.new : exchange
|
15
|
+
exchange_dest.publish(encode_message,
|
16
|
+
routing_key: routing_key || '',
|
17
|
+
content_type: options[:content_type] || content_type,
|
18
|
+
content_encoding: options[:content_encoding] || content_encoding,
|
19
|
+
type: options[:type] || type,
|
20
|
+
priority: options[:priority] || priority,
|
21
|
+
expiration: options[:expiration] || expiration,
|
22
|
+
headers: headers)
|
23
|
+
end
|
24
|
+
|
25
|
+
def message
|
26
|
+
@options
|
27
|
+
end
|
28
|
+
|
29
|
+
def encode_message
|
30
|
+
message_payload = message
|
31
|
+
message_payload = Legion::JSON.dump(message_payload) if message_payload.is_a? Hash
|
32
|
+
@options[:content_encoding] = 'identity'
|
33
|
+
return message_payload unless encrypt? && Legion::Settings[:crypt][:cs_encrypt_ready]
|
34
|
+
|
35
|
+
@options[:content_encoding] = 'encrypted/cs'
|
36
|
+
encrypt_message(message_payload)
|
37
|
+
end
|
38
|
+
|
39
|
+
def encrypt_message(message, _type = 'cs')
|
40
|
+
Legion::Crypt.encrypt(message)
|
41
|
+
end
|
42
|
+
|
43
|
+
def encrypt?
|
44
|
+
Legion::Settings[:transport][:messages][:encrypt]
|
45
|
+
end
|
46
|
+
|
47
|
+
def exchange_name
|
48
|
+
lex = self.class.ancestors.first.to_s.split('::')[2].downcase
|
49
|
+
"Legion::Extensions::#{lex.capitalize}::Transport::Exchanges::#{lex.capitalize}"
|
50
|
+
end
|
51
|
+
|
52
|
+
def exchange
|
53
|
+
Kernel.const_get(exchange_name)
|
54
|
+
end
|
55
|
+
|
56
|
+
def headers
|
57
|
+
@options[:headers] ||= {}
|
58
|
+
%i[task_id relationship_id trigger_namespace_id trigger_function_id parent_id master_id runner_namespace runner_class namespace_id function_id function chain_id debug].each do |header| # rubocop:disable Layout/LineLength
|
59
|
+
next unless @options.key? header
|
60
|
+
|
61
|
+
@options[:headers][header] = @options[header].to_s
|
62
|
+
end
|
63
|
+
@options[:headers]
|
64
|
+
rescue StandardError => e
|
65
|
+
Legion::Logging.error e.message
|
66
|
+
Legion::Logging.error e.backtrace
|
67
|
+
end
|
68
|
+
|
69
|
+
def priority
|
70
|
+
0
|
71
|
+
end
|
72
|
+
|
73
|
+
def expiration
|
74
|
+
nil
|
75
|
+
end
|
76
|
+
|
77
|
+
def content_type
|
78
|
+
'application/json'
|
79
|
+
end
|
80
|
+
|
81
|
+
def content_encoding
|
82
|
+
'identity'
|
83
|
+
end
|
84
|
+
|
85
|
+
def type
|
86
|
+
'task'
|
87
|
+
end
|
88
|
+
|
89
|
+
def timestamp
|
90
|
+
now.to_i
|
91
|
+
end
|
92
|
+
|
93
|
+
def validate
|
94
|
+
@valid = true
|
95
|
+
end
|
6
96
|
end
|
7
97
|
end
|
8
98
|
end
|
99
|
+
|
100
|
+
Dir[__dir__ + '/messages/*.rb'].sort.each { |file| require file }
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class CheckSubtask < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Task
|
9
|
+
end
|
10
|
+
|
11
|
+
def exchange_name
|
12
|
+
'Legion::Transport::Exchanges::Task'
|
13
|
+
end
|
14
|
+
|
15
|
+
def routing_key
|
16
|
+
'task.subtask.check'
|
17
|
+
end
|
18
|
+
|
19
|
+
def validate
|
20
|
+
@valid = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Legion::Transport::Messages
|
2
|
+
class Dynamic < Legion::Transport::Message
|
3
|
+
attr_accessor :options
|
4
|
+
def type
|
5
|
+
'task'
|
6
|
+
end
|
7
|
+
|
8
|
+
def message
|
9
|
+
{ args: @options[:args] || @options }
|
10
|
+
end
|
11
|
+
|
12
|
+
def routing_key
|
13
|
+
"#{function.runner.extension.values[:name]}.#{function.runner.values[:name]}.#{function.values[:name]}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def exchange
|
17
|
+
Legion::Transport::Exchange.new(function.runner.extension.values[:exchange])
|
18
|
+
end
|
19
|
+
|
20
|
+
def function
|
21
|
+
@function ||= Legion::Data::Model::Function[@options[:function_id]]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -4,32 +4,24 @@ module Legion
|
|
4
4
|
module Transport
|
5
5
|
module Messages
|
6
6
|
class LexRegister < Legion::Transport::Message
|
7
|
-
def initialize(namespace, method, options = {})
|
8
|
-
@namespace = namespace
|
9
|
-
@method = method
|
10
|
-
@options = options
|
11
|
-
end
|
12
|
-
|
13
7
|
def exchange
|
14
8
|
Legion::Transport::Exchanges::Lex
|
15
9
|
end
|
16
10
|
|
17
11
|
def routing_key
|
18
|
-
'lex.
|
12
|
+
'lex.register.save'
|
19
13
|
end
|
20
14
|
|
21
|
-
def
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
# raise unless method.is_a? String
|
32
|
-
# raise unless options.is_a? Hash
|
15
|
+
def validate
|
16
|
+
unless @options[:runner_namespace].is_a? String
|
17
|
+
# raise "runner_namespace is a #{@options[:runner_namespace].class}"
|
18
|
+
end
|
19
|
+
unless @options[:extension_namespace].is_a? String
|
20
|
+
# raise "extension_namespace is a #{@options[:extension_namespace].class}"
|
21
|
+
end
|
22
|
+
unless @options[:function].is_a?(String) || @options[:function].is_a?(Symbol)
|
23
|
+
# raise "function is a #{@options[:function].class}"
|
24
|
+
end
|
33
25
|
|
34
26
|
@valid = true
|
35
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'legion/transport/exchanges/lex'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class NodeHealth < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Node
|
9
|
+
end
|
10
|
+
|
11
|
+
def routing_key
|
12
|
+
'health'
|
13
|
+
end
|
14
|
+
|
15
|
+
def validate
|
16
|
+
@valid = true
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Legion::Transport::Messages
|
2
|
+
class RequestClusterSecret < Legion::Transport::Message
|
3
|
+
def routing_key
|
4
|
+
'node.crypt.push_cluster_secret'
|
5
|
+
end
|
6
|
+
|
7
|
+
def message
|
8
|
+
{ function: 'push_cluster_secret',
|
9
|
+
node_name: Legion::Settings[:client][:name],
|
10
|
+
queue_name: "node.#{Legion::Settings['client']['name']}",
|
11
|
+
runner_class: 'Legion::Extensions::Node::Runners::Crypt',
|
12
|
+
public_key: Base64.encode64(Legion::Crypt.public_key) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def exchange
|
16
|
+
require 'legion/transport/exchanges/node'
|
17
|
+
Legion::Transport::Exchanges::Node
|
18
|
+
end
|
19
|
+
|
20
|
+
def encrypt?
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
def type
|
25
|
+
'task'
|
26
|
+
end
|
27
|
+
|
28
|
+
def validate
|
29
|
+
@valid = true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class SubTask < Legion::Transport::Message
|
7
|
+
def exchange
|
8
|
+
Legion::Transport::Exchanges::Task
|
9
|
+
end
|
10
|
+
|
11
|
+
def message
|
12
|
+
{
|
13
|
+
transformation: @options[:transformation] || '{}',
|
14
|
+
conditions: @options[:conditions] || '{}',
|
15
|
+
results: @options[:results] || '{}'
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
def routing_key
|
20
|
+
if @options[:conditions].is_a?(String) && @options[:conditions].length > 2
|
21
|
+
'task.subtask.conditioner'
|
22
|
+
elsif @options[:transformation].is_a?(String) && @options[:transformation].length > 2
|
23
|
+
'task.subtask.transform'
|
24
|
+
elsif @options[:function_id].is_a? Integer
|
25
|
+
function = Legion::Data::Model::Function[@options[:function_id]]
|
26
|
+
"#{function.runner.extension.values[:exchange]}.#{function.runner.values[:queue]}.#{function.values[:name]}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def validate
|
31
|
+
raise TypeError unless @options[:function].is_a? String
|
32
|
+
|
33
|
+
@valid = true
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'legion/transport/exchanges/task'
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Transport
|
5
|
+
module Messages
|
6
|
+
class TaskLog < Legion::Transport::Message
|
7
|
+
def routing_key
|
8
|
+
"task.logs.create.#{@options[:task_id]}"
|
9
|
+
end
|
10
|
+
|
11
|
+
def exchange
|
12
|
+
Legion::Transport::Exchanges::Task
|
13
|
+
end
|
14
|
+
|
15
|
+
def message
|
16
|
+
@options[:function] = 'add_log'
|
17
|
+
@options[:runner_class] = 'Legion::Extensions::Tasker::Runners::Log'
|
18
|
+
@options
|
19
|
+
end
|
20
|
+
|
21
|
+
def generate_task?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate
|
26
|
+
@options[:task_id] = @options[:task_id].to_i if @options[:task_id].is_a? String
|
27
|
+
unless @options[:task_id].is_a? Integer
|
28
|
+
raise "task_id must be an integer but is #{@options[:task_id].class}(#{@options[:task_id]})"
|
29
|
+
end
|
30
|
+
|
31
|
+
@valid = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -11,15 +11,6 @@ module Legion
|
|
11
11
|
module Transport
|
12
12
|
module Messages
|
13
13
|
class TaskUpdate < Legion::Transport::Message
|
14
|
-
attr_accessor :status
|
15
|
-
attr_reader :task_id, :valid
|
16
|
-
def initialize(task_id, status = 'unknown', _options = {})
|
17
|
-
@status = status
|
18
|
-
@task_id = task_id
|
19
|
-
@routing_key = 'task.update'
|
20
|
-
validate
|
21
|
-
end
|
22
|
-
|
23
14
|
def routing_key
|
24
15
|
'task.update'
|
25
16
|
end
|
@@ -28,22 +19,11 @@ module Legion
|
|
28
19
|
Legion::Transport::Exchanges::Task
|
29
20
|
end
|
30
21
|
|
31
|
-
def message(status = @status, task_id = @task_id, options = {})
|
32
|
-
Legion::JSON.dump(args: { status: status, task_id: task_id, options: options })
|
33
|
-
end
|
34
|
-
|
35
|
-
def validate(status = @status, task_id = @task_id)
|
36
|
-
raise Legion::Exception::InvalidTaskId unless task_id.is_a? Integer
|
37
|
-
raise Legion::Exception::InvalidTaskStatus unless valid_status.include? status
|
38
|
-
|
39
|
-
@valid = true
|
40
|
-
end
|
41
|
-
|
42
22
|
def valid_status
|
43
23
|
conditioner = ['conditioner.queued', 'conditioner.failed', 'conditioner.exception']
|
44
|
-
|
45
|
-
task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception']
|
46
|
-
status = conditioner +
|
24
|
+
transformer = ['transformer.queued', 'transformer.succeeded', 'transformer.exception']
|
25
|
+
task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception', 'task.delayed']
|
26
|
+
status = conditioner + transformer + task
|
47
27
|
status
|
48
28
|
end
|
49
29
|
end
|