legionio 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/.gitignore +1 -1
- data/.rubocop.yml +42 -9
- data/CHANGELOG.md +9 -0
- data/Gemfile +0 -12
- data/Rakefile +1 -1
- data/bin/legion +6 -17
- data/exe/legion +10 -18
- data/legion.gemspec +9 -10
- data/lib/legion.rb +2 -2
- data/lib/legion/exceptions/invalidjson.rb +0 -3
- data/lib/legion/exceptions/missingargument.rb +0 -2
- data/lib/legion/exceptions/wrongtypes/array.rb +0 -3
- data/lib/legion/exceptions/wrongtypes/hash.rb +0 -3
- data/lib/legion/exceptions/wrongtypes/integer.rb +0 -3
- data/lib/legion/exceptions/wrongtypes/string.rb +0 -3
- data/lib/legion/extensions.rb +25 -12
- data/lib/legion/extensions/actors/base.rb +2 -6
- data/lib/legion/extensions/actors/every.rb +0 -2
- data/lib/legion/extensions/actors/loop.rb +0 -2
- data/lib/legion/extensions/actors/once.rb +0 -2
- data/lib/legion/extensions/actors/poll.rb +1 -3
- data/lib/legion/extensions/actors/subscription.rb +20 -10
- data/lib/legion/extensions/builders/actors.rb +4 -5
- data/lib/legion/extensions/builders/base.rb +0 -2
- data/lib/legion/extensions/builders/helpers.rb +0 -2
- data/lib/legion/extensions/builders/runners.rb +2 -3
- data/lib/legion/extensions/core.rb +15 -5
- data/lib/legion/extensions/data.rb +10 -0
- data/lib/legion/extensions/helpers/base.rb +1 -7
- data/lib/legion/extensions/helpers/core.rb +24 -3
- data/lib/legion/extensions/helpers/lex.rb +0 -2
- data/lib/legion/extensions/helpers/logger.rb +3 -4
- data/lib/legion/extensions/helpers/task.rb +1 -3
- data/lib/legion/extensions/helpers/transport.rb +0 -2
- data/lib/legion/extensions/transport.rb +4 -5
- data/lib/legion/process.rb +1 -13
- data/lib/legion/runner.rb +1 -3
- data/lib/legion/runner/log.rb +0 -2
- data/lib/legion/runner/status.rb +0 -3
- data/lib/legion/service.rb +18 -20
- data/lib/legion/{supervison.rb → supervision.rb} +1 -0
- data/lib/legion/version.rb +1 -3
- data/settings/client.json +14 -0
- metadata +22 -49
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require 'hashdiff'
|
4
2
|
|
5
3
|
module Legion
|
@@ -9,16 +7,14 @@ module Legion
|
|
9
7
|
include Legion::Extensions::Helpers::Lex
|
10
8
|
|
11
9
|
def runner
|
12
|
-
|
13
|
-
Legion::Runner.run(runner_class: runner_class, function: function, check_subtask: check_subtask, generate_task: generate_task)
|
10
|
+
Legion::Runner.run(runner_class: runner_class, function: function, check_subtask: check_subtask?, generate_task: generate_task?)
|
14
11
|
rescue StandardError => e
|
15
12
|
Legion::Logging.error e.message
|
16
13
|
Legion::Logging.error e.backtrace
|
17
14
|
end
|
18
15
|
|
19
16
|
def manual
|
20
|
-
|
21
|
-
runner_class.send(runner_function, args)
|
17
|
+
runner_class.send(runner_function, **args)
|
22
18
|
rescue StandardError => e
|
23
19
|
Legion::Logging.error e.message
|
24
20
|
Legion::Logging.error e.backtrace
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require_relative 'base'
|
4
2
|
require 'hashdiff'
|
5
3
|
require 'time'
|
@@ -51,7 +49,7 @@ module Legion
|
|
51
49
|
end
|
52
50
|
|
53
51
|
def cache_name
|
54
|
-
lex_name
|
52
|
+
"#{lex_name}_#{runner_name}"
|
55
53
|
end
|
56
54
|
|
57
55
|
def int_percentage_normalize
|
@@ -1,6 +1,5 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require_relative 'base'
|
2
|
+
require 'date'
|
4
3
|
|
5
4
|
module Legion
|
6
5
|
module Extensions
|
@@ -37,8 +36,12 @@ module Legion
|
|
37
36
|
end
|
38
37
|
|
39
38
|
def cancel
|
40
|
-
|
39
|
+
return true unless @queue.channel.active
|
40
|
+
|
41
|
+
log.debug "Closing subscription to #{@queue.name}"
|
42
|
+
@consumer.cancel
|
41
43
|
@queue.channel.close
|
44
|
+
true
|
42
45
|
end
|
43
46
|
|
44
47
|
def block
|
@@ -61,10 +64,9 @@ module Legion
|
|
61
64
|
true
|
62
65
|
end
|
63
66
|
|
64
|
-
def process_message(message, metadata)
|
67
|
+
def process_message(message, metadata, delivery_info)
|
65
68
|
payload = if metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/cs'
|
66
|
-
|
67
|
-
Legion::Crypt.decrypt(message)
|
69
|
+
Legion::Crypt.decrypt(message, metadata[:headers]['iv'])
|
68
70
|
elsif metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/pk'
|
69
71
|
Legion::Crypt.decrypt_from_keypair(metadata[:headers][:public_key], message)
|
70
72
|
else
|
@@ -74,10 +76,15 @@ module Legion
|
|
74
76
|
message = if metadata[:content_type] == 'application/json'
|
75
77
|
Legion::JSON.load(payload)
|
76
78
|
else
|
77
|
-
payload
|
79
|
+
{ value: payload }
|
78
80
|
end
|
81
|
+
if include_metadata_in_message?
|
82
|
+
message = message.merge(metadata[:headers].transform_keys(&:to_sym))
|
83
|
+
message[:routing_key] = delivery_info[:routing_key]
|
84
|
+
end
|
79
85
|
|
80
|
-
message = message
|
86
|
+
message[:timestamp] = (message[:timestamp_in_ms] / 1000).round if message.key?(:timestamp_in_ms) && !message.key?(:timestamp)
|
87
|
+
message[:datetime] = Time.at(message[:timestamp].to_i).to_datetime.to_s if message.key?(:timestamp)
|
81
88
|
message
|
82
89
|
end
|
83
90
|
|
@@ -94,8 +101,9 @@ module Legion
|
|
94
101
|
require 'legion/extensions/tasker/runners/updater'
|
95
102
|
sleep(delay_start)
|
96
103
|
consumer_tag = "#{Legion::Settings[:client][:name]}_#{lex_name}_#{runner_name}_#{Thread.current.object_id}"
|
97
|
-
|
98
|
-
|
104
|
+
on_cancellation = block { cancel }
|
105
|
+
@consumer = @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag, on_cancellation: on_cancellation) do |delivery_info, metadata, payload|
|
106
|
+
message = process_message(payload, metadata, delivery_info)
|
99
107
|
if use_runner?
|
100
108
|
Legion::Runner.run(**message,
|
101
109
|
runner_class: runner_class,
|
@@ -106,6 +114,8 @@ module Legion
|
|
106
114
|
runner_class.send(find_function(message), **message)
|
107
115
|
end
|
108
116
|
@queue.acknowledge(delivery_info.delivery_tag) if manual_ack
|
117
|
+
|
118
|
+
cancel if Legion::Settings[:client][:shutting_down]
|
109
119
|
rescue StandardError => e
|
110
120
|
Legion::Logging.error e.message
|
111
121
|
Legion::Logging.error e.backtrace
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require_relative 'base'
|
4
2
|
|
5
3
|
module Legion
|
@@ -9,6 +7,7 @@ module Legion
|
|
9
7
|
include Legion::Extensions::Builder::Base
|
10
8
|
|
11
9
|
attr_reader :actors
|
10
|
+
|
12
11
|
def build_actors
|
13
12
|
@actors = {}
|
14
13
|
require_files(actor_files)
|
@@ -19,7 +18,7 @@ module Legion
|
|
19
18
|
def build_actor_list
|
20
19
|
actor_files.each do |file|
|
21
20
|
actor_name = file.split('/').last.sub('.rb', '')
|
22
|
-
actor_class = lex_class
|
21
|
+
actor_class = "#{lex_class}::Actor::#{actor_name.split('_').collect(&:capitalize).join}"
|
23
22
|
@actors[actor_name.to_sym] = {
|
24
23
|
extension: lex_class.to_s.downcase,
|
25
24
|
extension_name: extension_name,
|
@@ -34,7 +33,7 @@ module Legion
|
|
34
33
|
@runners.each do |runner, attr|
|
35
34
|
next if @actors[runner.to_sym].is_a? Hash
|
36
35
|
|
37
|
-
actor_class = attr[:extension_class]
|
36
|
+
actor_class = "#{attr[:extension_class]}::Actor::#{runner.to_s.split('_').collect(&:capitalize).join}"
|
38
37
|
build_meta_actor(runner, attr) unless Kernel.const_defined? actor_class
|
39
38
|
@actors[runner.to_sym] = {
|
40
39
|
extension: attr[:extension],
|
@@ -49,7 +48,7 @@ module Legion
|
|
49
48
|
def build_meta_actor(runner, attr)
|
50
49
|
define_constant_two('Actor', root: lex_class)
|
51
50
|
|
52
|
-
Kernel.const_get(attr[:extension_class]
|
51
|
+
Kernel.const_get("#{attr[:extension_class]}::Actor")
|
53
52
|
.const_set(runner.to_s.split('_').collect(&:capitalize).join, Class.new(Legion::Extensions::Actors::Subscription))
|
54
53
|
end
|
55
54
|
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require_relative 'base'
|
4
2
|
|
5
3
|
module Legion
|
@@ -9,6 +7,7 @@ module Legion
|
|
9
7
|
include Legion::Extensions::Builder::Base
|
10
8
|
|
11
9
|
attr_reader :runners
|
10
|
+
|
12
11
|
def build_runners
|
13
12
|
@runners = {}
|
14
13
|
lex_class.const_set('Runners', Module.new) unless lex_class.const_defined?('Runners')
|
@@ -19,7 +18,7 @@ module Legion
|
|
19
18
|
def build_runner_list
|
20
19
|
runner_files.each do |file|
|
21
20
|
runner_name = file.split('/').last.sub('.rb', '')
|
22
|
-
runner_class =
|
21
|
+
runner_class = "#{lex_class}::Runners::#{runner_name.split('_').collect(&:capitalize).join}"
|
23
22
|
loaded_runner = Kernel.const_get(runner_class)
|
24
23
|
@runners[runner_name.to_sym] = {
|
25
24
|
extension: lex_class.to_s.downcase,
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require_relative 'builders/actors'
|
4
2
|
require_relative 'builders/helpers'
|
5
3
|
require_relative 'builders/runners'
|
@@ -45,11 +43,11 @@ module Legion
|
|
45
43
|
end
|
46
44
|
|
47
45
|
def build_transport
|
48
|
-
if File.exist? extension_path
|
46
|
+
if File.exist? "#{extension_path}/transport/autobuild.rb"
|
49
47
|
require "#{extension_path}/transport/autobuild"
|
50
48
|
extension_class::Transport::AutoBuild.build
|
51
49
|
log.warn 'still using transport::autobuild, please upgrade'
|
52
|
-
elsif File.exist? extension_path
|
50
|
+
elsif File.exist? "#{extension_path}/transport.rb"
|
53
51
|
require "#{extension_path}/transport"
|
54
52
|
extension_class::Transport.build
|
55
53
|
else
|
@@ -58,7 +56,7 @@ module Legion
|
|
58
56
|
end
|
59
57
|
end
|
60
58
|
|
61
|
-
def build_settings
|
59
|
+
def build_settings # rubocop:disable Metrics/AbcSize
|
62
60
|
if Legion::Settings[:extensions].key?(lex_name.to_sym)
|
63
61
|
Legion::Settings[:default_extension_settings].each do |key, value|
|
64
62
|
Legion::Settings[:extensions][lex_name.to_sym][key.to_sym] = if Legion::Settings[:extensions][lex_name.to_sym].key?(key.to_sym)
|
@@ -70,6 +68,18 @@ module Legion
|
|
70
68
|
else
|
71
69
|
Legion::Settings[:extensions][lex_name.to_sym] = Legion::Settings[:default_extension_settings]
|
72
70
|
end
|
71
|
+
|
72
|
+
default_settings.each do |key, value|
|
73
|
+
Legion::Settings[:extensions][lex_name.to_sym][key.to_sym] = if Legion::Settings[:extensions][lex_name.to_sym].key?(key.to_sym)
|
74
|
+
value.merge(Legion::Settings[:extensions][lex_name.to_sym][key.to_sym])
|
75
|
+
else
|
76
|
+
value
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def default_settings
|
82
|
+
{}
|
73
83
|
end
|
74
84
|
|
75
85
|
def auto_generate_transport
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
module Legion
|
4
2
|
module Extensions
|
5
3
|
module Helpers
|
@@ -56,10 +54,6 @@ module Legion
|
|
56
54
|
end
|
57
55
|
alias extension_path full_path
|
58
56
|
|
59
|
-
def to_json(object)
|
60
|
-
Legion::JSON.dump(object)
|
61
|
-
end
|
62
|
-
|
63
57
|
def from_json(string)
|
64
58
|
Legion::JSON.load(string)
|
65
59
|
end
|
@@ -76,7 +70,7 @@ module Legion
|
|
76
70
|
hash.each_with_object({}) do |(k, v), ret|
|
77
71
|
key = recursive_key + k.to_s
|
78
72
|
if v.is_a? Hash
|
79
|
-
ret.merge! to_dotted_hash(v, key
|
73
|
+
ret.merge! to_dotted_hash(v, "#{key}.")
|
80
74
|
else
|
81
75
|
ret[key.to_sym] = v
|
82
76
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require_relative 'base'
|
4
2
|
module Legion
|
5
3
|
module Extensions
|
@@ -11,8 +9,31 @@ module Legion
|
|
11
9
|
if Legion::Settings[:extensions].key?(lex_filename.to_sym)
|
12
10
|
Legion::Settings[:extensions][lex_filename.to_sym]
|
13
11
|
else
|
14
|
-
{ logger: { level: 'info', extended: false } }
|
12
|
+
{ logger: { level: 'info', extended: false, internal: false } }
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# looks local, then in crypt, then settings, then cache, then env
|
17
|
+
def find_setting(name, **opts)
|
18
|
+
log.debug ".find_setting(#{name}) called"
|
19
|
+
return opts[name.to_sym] if opts.key? name.to_sym
|
20
|
+
|
21
|
+
string_name = "#{lex_name}_#{name.to_s.downcase}"
|
22
|
+
if Legion::Settings[:crypt][:vault][:connected] && Legion::Crypt.exist?(lex_name)
|
23
|
+
log.debug "looking for #{string_name} in Legion::Crypt"
|
24
|
+
crypt_result = Legion::Crypt.get(lex_name)
|
25
|
+
return crypt_result[name.to_sym] if crypt_result.is_a?(Hash) && crypt_result.key?(name.to_sym)
|
26
|
+
end
|
27
|
+
return settings[name.to_sym] if settings.key? name.to_sym
|
28
|
+
|
29
|
+
if Legion::Settings[:cache][:connected]
|
30
|
+
log.debug "looking for #{string_name} in Legion::Cache"
|
31
|
+
cache_result = Legion::Cache.get(string_name)
|
32
|
+
return cache_result unless cache_result.nil?
|
15
33
|
end
|
34
|
+
|
35
|
+
ENV[string_name] if ENV.key? string_name
|
36
|
+
nil
|
16
37
|
end
|
17
38
|
end
|
18
39
|
end
|
@@ -1,10 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
module Legion
|
4
2
|
module Extensions
|
5
3
|
module Helpers
|
6
4
|
module Logger
|
7
|
-
def log
|
5
|
+
def log # rubocop:disable Metrics/AbcSize
|
8
6
|
return @log unless @log.nil?
|
9
7
|
|
10
8
|
logger_hash = { lex: lex_filename || nil }
|
@@ -12,6 +10,7 @@ module Legion
|
|
12
10
|
if respond_to?(:settings) && settings.key?(:logger)
|
13
11
|
logger_hash[:level] = settings[:logger][:level] if settings[:logger].key? :level
|
14
12
|
logger_hash[:level] = settings[:logger][:log_file] if settings[:logger].key? :log_file
|
13
|
+
logger_hash[:level] = 'info' unless settings[:logger].key? :log_file
|
15
14
|
logger_hash[:trace] = settings[:logger][:trace] if settings[:logger].key? :trace
|
16
15
|
logger_hash[:extended] = settings[:logger][:extended] if settings[:logger].key? :extended
|
17
16
|
elsif respond_to?(:settings)
|
@@ -20,7 +19,7 @@ module Legion
|
|
20
19
|
else
|
21
20
|
Legion::Logging.warn 'no settings'
|
22
21
|
end
|
23
|
-
@log = Legion::Logging::Logger.new(logger_hash)
|
22
|
+
@log = Legion::Logging::Logger.new(**logger_hash)
|
24
23
|
end
|
25
24
|
|
26
25
|
def handle_exception(exception, task_id: nil, **opts)
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require 'legion/transport'
|
4
2
|
require 'legion/transport/messages/task_update'
|
5
3
|
require 'legion/transport/messages/task_log'
|
@@ -19,7 +17,7 @@ module Legion
|
|
19
17
|
%i[results payload function_args payload results].each do |column|
|
20
18
|
update_hash[column] = opts[column] if opts.key? column
|
21
19
|
end
|
22
|
-
Legion::Transport::Messages::TaskUpdate.new(update_hash).publish
|
20
|
+
Legion::Transport::Messages::TaskUpdate.new(**update_hash).publish
|
23
21
|
rescue StandardError => e
|
24
22
|
log.fatal e.message
|
25
23
|
log.fatal e.backtrace
|
@@ -1,11 +1,10 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
module Legion
|
4
2
|
module Extensions
|
5
3
|
module Transport
|
6
4
|
include Legion::Extensions::Helpers::Transport
|
7
5
|
|
8
6
|
attr_accessor :exchanges, :queues, :consumers, :messages
|
7
|
+
|
9
8
|
def build
|
10
9
|
@queues = []
|
11
10
|
@exchanges = []
|
@@ -43,7 +42,7 @@ module Legion
|
|
43
42
|
end
|
44
43
|
end
|
45
44
|
|
46
|
-
def auto_create_exchange(exchange, default_exchange = false)
|
45
|
+
def auto_create_exchange(exchange, default_exchange = false) # rubocop:disable Style/OptionalBooleanParameter
|
47
46
|
if Object.const_defined? exchange
|
48
47
|
Legion::Logging.warn "#{exchange} is already defined"
|
49
48
|
return
|
@@ -63,7 +62,7 @@ module Legion
|
|
63
62
|
return
|
64
63
|
end
|
65
64
|
|
66
|
-
|
65
|
+
transport_class::Queues.const_set(queue.split('::').last, Class.new(Legion::Transport::Queue))
|
67
66
|
end
|
68
67
|
|
69
68
|
def auto_create_dlx_exchange
|
@@ -82,7 +81,7 @@ module Legion
|
|
82
81
|
return if transport_class::Queues.const_defined?('Dlx')
|
83
82
|
|
84
83
|
special_name = default_exchange.new.exchange_name
|
85
|
-
dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx"
|
84
|
+
dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx", auto_delete: false
|
86
85
|
dlx_queue.bind("#{special_name}.dlx", { routing_key: '#' })
|
87
86
|
end
|
88
87
|
|
data/lib/legion/process.rb
CHANGED
@@ -1,9 +1,6 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
1
|
require 'fileutils'
|
4
2
|
|
5
3
|
module Legion
|
6
|
-
# Responsible for starting the Legion process
|
7
4
|
class Process
|
8
5
|
def self.run!(options)
|
9
6
|
Legion::Process.new(options).run!
|
@@ -51,7 +48,7 @@ module Legion
|
|
51
48
|
trap_signals
|
52
49
|
|
53
50
|
until quit
|
54
|
-
sleep(1)
|
51
|
+
sleep(1)
|
55
52
|
if @options.key? :time_limit
|
56
53
|
@quit = true if Time.now - start_time > @options[:time_limit]
|
57
54
|
end
|
@@ -113,16 +110,7 @@ module Legion
|
|
113
110
|
:not_owned
|
114
111
|
end
|
115
112
|
|
116
|
-
#==========================================================================
|
117
|
-
# SIGNAL HANDLING
|
118
|
-
#==========================================================================
|
119
|
-
|
120
113
|
def trap_signals
|
121
|
-
# trap(:QUIT) do # graceful shutdown
|
122
|
-
# info 'shutting down'
|
123
|
-
# @quit = true
|
124
|
-
# end
|
125
|
-
|
126
114
|
trap('SIGTERM') do
|
127
115
|
info 'sigterm'
|
128
116
|
end
|