legion-transport 0.1.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +12 -12
  3. data/.idea/.rakeTasks +7 -0
  4. data/.idea/legion-transport.iml +45 -0
  5. data/.idea/misc.xml +7 -0
  6. data/.idea/modules.xml +8 -0
  7. data/.idea/vagrant.xml +7 -0
  8. data/.idea/workspace.xml +14 -0
  9. data/.rspec +1 -0
  10. data/.rubocop.yml +1 -8
  11. data/CHANGELOG.md +0 -0
  12. data/Gemfile +0 -1
  13. data/README.md +35 -0
  14. data/legion-transport.gemspec +15 -5
  15. data/lib/legion/transport.rb +11 -9
  16. data/lib/legion/transport/common.rb +23 -18
  17. data/lib/legion/transport/connection.rb +55 -12
  18. data/lib/legion/transport/consumer.rb +7 -1
  19. data/lib/legion/transport/exchange.rb +44 -6
  20. data/lib/legion/transport/exchanges/crypt.rb +11 -0
  21. data/lib/legion/transport/exchanges/node.rb +11 -0
  22. data/lib/legion/transport/message.rb +94 -2
  23. data/lib/legion/transport/messages/check_subtask.rb +25 -0
  24. data/lib/legion/transport/messages/dynamic.rb +24 -0
  25. data/lib/legion/transport/messages/lex_register.rb +11 -19
  26. data/lib/legion/transport/messages/node_health.rb +21 -0
  27. data/lib/legion/transport/messages/request_cluster_secret.rb +32 -0
  28. data/lib/legion/transport/messages/subtask.rb +38 -0
  29. data/lib/legion/transport/messages/task_log.rb +36 -0
  30. data/lib/legion/transport/messages/task_update.rb +3 -23
  31. data/lib/legion/transport/queue.rb +63 -6
  32. data/lib/legion/transport/queues/node.rb +15 -0
  33. data/lib/legion/transport/queues/node_status.rb +1 -1
  34. data/lib/legion/transport/queues/task_log.rb +1 -1
  35. data/lib/legion/transport/queues/task_update.rb +1 -1
  36. data/lib/legion/transport/settings.rb +64 -0
  37. data/lib/legion/transport/version.rb +1 -1
  38. data/settings/transport.json +9 -1
  39. metadata +81 -33
  40. data/lib/legion/transport/connections/bunny.rb +0 -53
  41. data/lib/legion/transport/connections/common.rb +0 -36
  42. data/lib/legion/transport/connections/marchhare.rb +0 -20
  43. data/lib/legion/transport/consumers/bunny.rb +0 -11
  44. data/lib/legion/transport/consumers/common.rb +0 -8
  45. data/lib/legion/transport/consumers/marchhare.rb +0 -11
  46. data/lib/legion/transport/exchanges/bunny.rb +0 -28
  47. data/lib/legion/transport/exchanges/common.rb +0 -32
  48. data/lib/legion/transport/exchanges/marchhare.rb +0 -17
  49. data/lib/legion/transport/messages/base.rb +0 -22
  50. data/lib/legion/transport/messages/node_status.rb +0 -49
  51. data/lib/legion/transport/messages/task_check_subtask.rb +0 -35
  52. data/lib/legion/transport/messages/task_subtask.rb +0 -45
  53. data/lib/legion/transport/queues/bunny.rb +0 -26
  54. data/lib/legion/transport/queues/common.rb +0 -45
  55. data/lib/legion/transport/queues/marchhare.rb +0 -26
@@ -1,6 +1,12 @@
1
1
  module Legion
2
2
  module Transport
3
- class Connection
3
+ class Consumer < Legion::Transport::CONNECTOR::Consumer
4
+ include Legion::Transport::Common
5
+ attr_reader :consumer_tag
6
+ def initialize(queue:, no_ack: false, exclusive: false, consumer_tag: generate_consumer_tag, **opts)
7
+ @consumer_tag = consumer_tag
8
+ super(channel, queue, consumer_tag, no_ack, exclusive, opts)
9
+ end
4
10
  end
5
11
  end
6
12
  end
@@ -1,12 +1,50 @@
1
1
  module Legion
2
2
  module Transport
3
- if RUBY_ENGINE == 'jruby'
4
- require 'legion/transport/exchanges/marchhare'
5
- class Exchange < Legion::Transport::Exchanges::Marchhare
3
+ class Exchange < Legion::Transport::CONNECTOR::Exchange
4
+ include Legion::Transport::Common
5
+
6
+ def initialize(exchange = exchange_name, options = {})
7
+ @options = options
8
+ @type = options[:type] || default_type
9
+ super(channel, @type, exchange, options_builder(default_options, exchange_options, @options))
10
+ rescue ::Bunny::PreconditionFailed, ::Bunny::ChannelAlreadyClosed
11
+ raise unless @retries.nil?
12
+
13
+ @retries = 1
14
+ delete_exchange(exchange)
15
+ retry
6
16
  end
7
- else
8
- require 'legion/transport/exchanges/bunny'
9
- class Exchange < Legion::Transport::Exchanges::Bunny
17
+
18
+ def delete_exchange(exchange)
19
+ Legion::Logging.warn "Exchange:#{exchange} exists with wrong parameters, deleting and creating"
20
+ channel.exchange_delete(exchange)
21
+ end
22
+
23
+ def default_options
24
+ hash = {}
25
+ hash[:durable] = true
26
+ hash[:auto_delete] = false
27
+ hash[:arguments] = {}
28
+ hash
29
+ end
30
+
31
+ def exchange_name
32
+ self.class.ancestors.first.to_s.split('::')[2].downcase
33
+ end
34
+
35
+ def exchange_options
36
+ {}
37
+ end
38
+
39
+ def delete(options = {})
40
+ super(options)
41
+ true
42
+ rescue ::Bunny::PreconditionFailed
43
+ false
44
+ end
45
+
46
+ def default_type
47
+ 'topic'
10
48
  end
11
49
  end
12
50
  end
@@ -0,0 +1,11 @@
1
+ module Legion
2
+ module Transport
3
+ module Exchanges
4
+ class Lex < Legion::Transport::Exchange
5
+ def exchange_name
6
+ 'crypt'
7
+ end
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,11 @@
1
+ module Legion
2
+ module Transport
3
+ module Exchanges
4
+ class Node < Legion::Transport::Exchange
5
+ def exchange_name
6
+ 'node'
7
+ end
8
+ end
9
+ end
10
+ end
11
+ end
@@ -1,8 +1,100 @@
1
1
  module Legion
2
2
  module Transport
3
3
  class Message
4
- require 'legion/transport/messages/base'
5
- include Legion::Transport::Messages::Base
4
+ include Legion::Transport::Common
5
+
6
+ def initialize(**options)
7
+ @options = options
8
+ validate
9
+ end
10
+
11
+ def publish(options = @options)
12
+ raise unless @valid
13
+
14
+ exchange_dest = exchange.respond_to?(:new) ? exchange.new : exchange
15
+ exchange_dest.publish(encode_message,
16
+ routing_key: routing_key || '',
17
+ content_type: options[:content_type] || content_type,
18
+ content_encoding: options[:content_encoding] || content_encoding,
19
+ type: options[:type] || type,
20
+ priority: options[:priority] || priority,
21
+ expiration: options[:expiration] || expiration,
22
+ headers: headers)
23
+ end
24
+
25
+ def message
26
+ @options
27
+ end
28
+
29
+ def encode_message
30
+ message_payload = message
31
+ message_payload = Legion::JSON.dump(message_payload) if message_payload.is_a? Hash
32
+ @options[:content_encoding] = 'identity'
33
+ return message_payload unless encrypt? && Legion::Settings[:crypt][:cs_encrypt_ready]
34
+
35
+ @options[:content_encoding] = 'encrypted/cs'
36
+ encrypt_message(message_payload)
37
+ end
38
+
39
+ def encrypt_message(message, _type = 'cs')
40
+ Legion::Crypt.encrypt(message)
41
+ end
42
+
43
+ def encrypt?
44
+ Legion::Settings[:transport][:messages][:encrypt]
45
+ end
46
+
47
+ def exchange_name
48
+ lex = self.class.ancestors.first.to_s.split('::')[2].downcase
49
+ "Legion::Extensions::#{lex.capitalize}::Transport::Exchanges::#{lex.capitalize}"
50
+ end
51
+
52
+ def exchange
53
+ Kernel.const_get(exchange_name)
54
+ end
55
+
56
+ def headers
57
+ @options[:headers] ||= {}
58
+ %i[task_id relationship_id trigger_namespace_id trigger_function_id parent_id master_id runner_namespace runner_class namespace_id function_id function chain_id debug].each do |header| # rubocop:disable Layout/LineLength
59
+ next unless @options.key? header
60
+
61
+ @options[:headers][header] = @options[header].to_s
62
+ end
63
+ @options[:headers]
64
+ rescue StandardError => e
65
+ Legion::Logging.error e.message
66
+ Legion::Logging.error e.backtrace
67
+ end
68
+
69
+ def priority
70
+ 0
71
+ end
72
+
73
+ def expiration
74
+ nil
75
+ end
76
+
77
+ def content_type
78
+ 'application/json'
79
+ end
80
+
81
+ def content_encoding
82
+ 'identity'
83
+ end
84
+
85
+ def type
86
+ 'task'
87
+ end
88
+
89
+ def timestamp
90
+ now.to_i
91
+ end
92
+
93
+ def validate
94
+ @valid = true
95
+ end
6
96
  end
7
97
  end
8
98
  end
99
+
100
+ Dir[__dir__ + '/messages/*.rb'].sort.each { |file| require file }
@@ -0,0 +1,25 @@
1
+ require 'legion/transport/exchanges/task'
2
+
3
+ module Legion
4
+ module Transport
5
+ module Messages
6
+ class CheckSubtask < Legion::Transport::Message
7
+ def exchange
8
+ Legion::Transport::Exchanges::Task
9
+ end
10
+
11
+ def exchange_name
12
+ 'Legion::Transport::Exchanges::Task'
13
+ end
14
+
15
+ def routing_key
16
+ 'task.subtask.check'
17
+ end
18
+
19
+ def validate
20
+ @valid = true
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,24 @@
1
+ module Legion::Transport::Messages
2
+ class Dynamic < Legion::Transport::Message
3
+ attr_accessor :options
4
+ def type
5
+ 'task'
6
+ end
7
+
8
+ def message
9
+ { args: @options[:args] || @options }
10
+ end
11
+
12
+ def routing_key
13
+ "#{function.runner.extension.values[:name]}.#{function.runner.values[:name]}.#{function.values[:name]}"
14
+ end
15
+
16
+ def exchange
17
+ Legion::Transport::Exchange.new(function.runner.extension.values[:exchange])
18
+ end
19
+
20
+ def function
21
+ @function ||= Legion::Data::Model::Function[@options[:function_id]]
22
+ end
23
+ end
24
+ end
@@ -4,32 +4,24 @@ module Legion
4
4
  module Transport
5
5
  module Messages
6
6
  class LexRegister < Legion::Transport::Message
7
- def initialize(namespace, method, options = {})
8
- @namespace = namespace
9
- @method = method
10
- @options = options
11
- end
12
-
13
7
  def exchange
14
8
  Legion::Transport::Exchanges::Lex
15
9
  end
16
10
 
17
11
  def routing_key
18
- 'lex.methods.register'
12
+ 'lex.register.save'
19
13
  end
20
14
 
21
- def message(namespace = @namespace, method = @method, options = @options)
22
- obj = { args: { namespace: namespace, method: method } }
23
- obj[:args][:namespace_queue] = options[:namespace][:queue] unless options[:namespace][:queue].nil?
24
- obj[:args][:namespace_uri] = options[:namespace][:uri] unless options[:namespace][:uri].nil?
25
- Legion::JSON.dump(obj)
26
- end
27
-
28
- def validate(namespace = @namespace, _method = @method, _options = @options)
29
- raise unless namespace.is_a? String
30
-
31
- # raise unless method.is_a? String
32
- # raise unless options.is_a? Hash
15
+ def validate
16
+ unless @options[:runner_namespace].is_a? String
17
+ # raise "runner_namespace is a #{@options[:runner_namespace].class}"
18
+ end
19
+ unless @options[:extension_namespace].is_a? String
20
+ # raise "extension_namespace is a #{@options[:extension_namespace].class}"
21
+ end
22
+ unless @options[:function].is_a?(String) || @options[:function].is_a?(Symbol)
23
+ # raise "function is a #{@options[:function].class}"
24
+ end
33
25
 
34
26
  @valid = true
35
27
  end
@@ -0,0 +1,21 @@
1
+ require 'legion/transport/exchanges/lex'
2
+
3
+ module Legion
4
+ module Transport
5
+ module Messages
6
+ class NodeHealth < Legion::Transport::Message
7
+ def exchange
8
+ Legion::Transport::Exchanges::Node
9
+ end
10
+
11
+ def routing_key
12
+ 'health'
13
+ end
14
+
15
+ def validate
16
+ @valid = true
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,32 @@
1
+ module Legion::Transport::Messages
2
+ class RequestClusterSecret < Legion::Transport::Message
3
+ def routing_key
4
+ 'node.crypt.push_cluster_secret'
5
+ end
6
+
7
+ def message
8
+ { function: 'push_cluster_secret',
9
+ node_name: Legion::Settings[:client][:name],
10
+ queue_name: "node.#{Legion::Settings['client']['name']}",
11
+ runner_class: 'Legion::Extensions::Node::Runners::Crypt',
12
+ public_key: Base64.encode64(Legion::Crypt.public_key) }
13
+ end
14
+
15
+ def exchange
16
+ require 'legion/transport/exchanges/node'
17
+ Legion::Transport::Exchanges::Node
18
+ end
19
+
20
+ def encrypt?
21
+ false
22
+ end
23
+
24
+ def type
25
+ 'task'
26
+ end
27
+
28
+ def validate
29
+ @valid = true
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,38 @@
1
+ require 'legion/transport/exchanges/task'
2
+
3
+ module Legion
4
+ module Transport
5
+ module Messages
6
+ class SubTask < Legion::Transport::Message
7
+ def exchange
8
+ Legion::Transport::Exchanges::Task
9
+ end
10
+
11
+ def message
12
+ {
13
+ transformation: @options[:transformation] || '{}',
14
+ conditions: @options[:conditions] || '{}',
15
+ results: @options[:results] || '{}'
16
+ }
17
+ end
18
+
19
+ def routing_key
20
+ if @options[:conditions].is_a?(String) && @options[:conditions].length > 2
21
+ 'task.subtask.conditioner'
22
+ elsif @options[:transformation].is_a?(String) && @options[:transformation].length > 2
23
+ 'task.subtask.transform'
24
+ elsif @options[:function_id].is_a? Integer
25
+ function = Legion::Data::Model::Function[@options[:function_id]]
26
+ "#{function.runner.extension.values[:exchange]}.#{function.runner.values[:queue]}.#{function.values[:name]}"
27
+ end
28
+ end
29
+
30
+ def validate
31
+ raise TypeError unless @options[:function].is_a? String
32
+
33
+ @valid = true
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,36 @@
1
+ require 'legion/transport/exchanges/task'
2
+
3
+ module Legion
4
+ module Transport
5
+ module Messages
6
+ class TaskLog < Legion::Transport::Message
7
+ def routing_key
8
+ "task.logs.create.#{@options[:task_id]}"
9
+ end
10
+
11
+ def exchange
12
+ Legion::Transport::Exchanges::Task
13
+ end
14
+
15
+ def message
16
+ @options[:function] = 'add_log'
17
+ @options[:runner_class] = 'Legion::Extensions::Tasker::Runners::Log'
18
+ @options
19
+ end
20
+
21
+ def generate_task?
22
+ false
23
+ end
24
+
25
+ def validate
26
+ @options[:task_id] = @options[:task_id].to_i if @options[:task_id].is_a? String
27
+ unless @options[:task_id].is_a? Integer
28
+ raise "task_id must be an integer but is #{@options[:task_id].class}(#{@options[:task_id]})"
29
+ end
30
+
31
+ @valid = true
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -11,15 +11,6 @@ module Legion
11
11
  module Transport
12
12
  module Messages
13
13
  class TaskUpdate < Legion::Transport::Message
14
- attr_accessor :status
15
- attr_reader :task_id, :valid
16
- def initialize(task_id, status = 'unknown', _options = {})
17
- @status = status
18
- @task_id = task_id
19
- @routing_key = 'task.update'
20
- validate
21
- end
22
-
23
14
  def routing_key
24
15
  'task.update'
25
16
  end
@@ -28,22 +19,11 @@ module Legion
28
19
  Legion::Transport::Exchanges::Task
29
20
  end
30
21
 
31
- def message(status = @status, task_id = @task_id, options = {})
32
- Legion::JSON.dump(args: { status: status, task_id: task_id, options: options })
33
- end
34
-
35
- def validate(status = @status, task_id = @task_id)
36
- raise Legion::Exception::InvalidTaskId unless task_id.is_a? Integer
37
- raise Legion::Exception::InvalidTaskStatus unless valid_status.include? status
38
-
39
- @valid = true
40
- end
41
-
42
22
  def valid_status
43
23
  conditioner = ['conditioner.queued', 'conditioner.failed', 'conditioner.exception']
44
- transfomer = ['transformer.queued', 'transformer.succeeded', 'transformer.exception']
45
- task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception']
46
- status = conditioner + transfomer + task
24
+ transformer = ['transformer.queued', 'transformer.succeeded', 'transformer.exception']
25
+ task = ['task.scheduled', 'task.queued', 'task.completed', 'task.exception', 'task.delayed']
26
+ status = conditioner + transformer + task
47
27
  status
48
28
  end
49
29
  end