legionio 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +21 -0
  3. data/.gitignore +1 -1
  4. data/.rubocop.yml +42 -9
  5. data/CHANGELOG.md +9 -0
  6. data/Gemfile +0 -12
  7. data/Rakefile +1 -1
  8. data/bin/legion +6 -17
  9. data/exe/legion +10 -18
  10. data/legion.gemspec +9 -10
  11. data/lib/legion.rb +2 -2
  12. data/lib/legion/exceptions/invalidjson.rb +0 -3
  13. data/lib/legion/exceptions/missingargument.rb +0 -2
  14. data/lib/legion/exceptions/wrongtypes/array.rb +0 -3
  15. data/lib/legion/exceptions/wrongtypes/hash.rb +0 -3
  16. data/lib/legion/exceptions/wrongtypes/integer.rb +0 -3
  17. data/lib/legion/exceptions/wrongtypes/string.rb +0 -3
  18. data/lib/legion/extensions.rb +25 -12
  19. data/lib/legion/extensions/actors/base.rb +2 -6
  20. data/lib/legion/extensions/actors/every.rb +0 -2
  21. data/lib/legion/extensions/actors/loop.rb +0 -2
  22. data/lib/legion/extensions/actors/once.rb +0 -2
  23. data/lib/legion/extensions/actors/poll.rb +1 -3
  24. data/lib/legion/extensions/actors/subscription.rb +20 -10
  25. data/lib/legion/extensions/builders/actors.rb +4 -5
  26. data/lib/legion/extensions/builders/base.rb +0 -2
  27. data/lib/legion/extensions/builders/helpers.rb +0 -2
  28. data/lib/legion/extensions/builders/runners.rb +2 -3
  29. data/lib/legion/extensions/core.rb +15 -5
  30. data/lib/legion/extensions/data.rb +10 -0
  31. data/lib/legion/extensions/helpers/base.rb +1 -7
  32. data/lib/legion/extensions/helpers/core.rb +24 -3
  33. data/lib/legion/extensions/helpers/lex.rb +0 -2
  34. data/lib/legion/extensions/helpers/logger.rb +3 -4
  35. data/lib/legion/extensions/helpers/task.rb +1 -3
  36. data/lib/legion/extensions/helpers/transport.rb +0 -2
  37. data/lib/legion/extensions/transport.rb +4 -5
  38. data/lib/legion/process.rb +1 -13
  39. data/lib/legion/runner.rb +1 -3
  40. data/lib/legion/runner/log.rb +0 -2
  41. data/lib/legion/runner/status.rb +0 -3
  42. data/lib/legion/service.rb +18 -20
  43. data/lib/legion/{supervison.rb → supervision.rb} +1 -0
  44. data/lib/legion/version.rb +1 -3
  45. data/settings/client.json +14 -0
  46. metadata +22 -49
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require 'hashdiff'
4
2
 
5
3
  module Legion
@@ -9,16 +7,14 @@ module Legion
9
7
  include Legion::Extensions::Helpers::Lex
10
8
 
11
9
  def runner
12
- # runner_class:, function:, task_id: nil, args: nil, check_subtask: true, generate_task: true, parent_id: nil, master_id: nil, **opts
13
- Legion::Runner.run(runner_class: runner_class, function: function, check_subtask: check_subtask, generate_task: generate_task)
10
+ Legion::Runner.run(runner_class: runner_class, function: function, check_subtask: check_subtask?, generate_task: generate_task?)
14
11
  rescue StandardError => e
15
12
  Legion::Logging.error e.message
16
13
  Legion::Logging.error e.backtrace
17
14
  end
18
15
 
19
16
  def manual
20
- # Legion::Runner.run(runner_class: runner_class, function: runner_function, check_subtask: false, generate_task: false, args: args)
21
- runner_class.send(runner_function, args)
17
+ runner_class.send(runner_function, **args)
22
18
  rescue StandardError => e
23
19
  Legion::Logging.error e.message
24
20
  Legion::Logging.error e.backtrace
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
  require 'hashdiff'
5
3
  require 'time'
@@ -51,7 +49,7 @@ module Legion
51
49
  end
52
50
 
53
51
  def cache_name
54
- lex_name.to_s + '_' + runner_name
52
+ "#{lex_name}_#{runner_name}"
55
53
  end
56
54
 
57
55
  def int_percentage_normalize
@@ -1,6 +1,5 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
2
+ require 'date'
4
3
 
5
4
  module Legion
6
5
  module Extensions
@@ -37,8 +36,12 @@ module Legion
37
36
  end
38
37
 
39
38
  def cancel
40
- Legion::Logging.info "Closing subscription to #{@queue.name}"
39
+ return true unless @queue.channel.active
40
+
41
+ log.debug "Closing subscription to #{@queue.name}"
42
+ @consumer.cancel
41
43
  @queue.channel.close
44
+ true
42
45
  end
43
46
 
44
47
  def block
@@ -61,10 +64,9 @@ module Legion
61
64
  true
62
65
  end
63
66
 
64
- def process_message(message, metadata)
67
+ def process_message(message, metadata, delivery_info)
65
68
  payload = if metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/cs'
66
- sleep(2) unless Legion::Settings[:crypt][:cs_encrypt_ready]
67
- Legion::Crypt.decrypt(message)
69
+ Legion::Crypt.decrypt(message, metadata[:headers]['iv'])
68
70
  elsif metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/pk'
69
71
  Legion::Crypt.decrypt_from_keypair(metadata[:headers][:public_key], message)
70
72
  else
@@ -74,10 +76,15 @@ module Legion
74
76
  message = if metadata[:content_type] == 'application/json'
75
77
  Legion::JSON.load(payload)
76
78
  else
77
- payload
79
+ { value: payload }
78
80
  end
81
+ if include_metadata_in_message?
82
+ message = message.merge(metadata[:headers].transform_keys(&:to_sym))
83
+ message[:routing_key] = delivery_info[:routing_key]
84
+ end
79
85
 
80
- message = message.merge(metadata[:headers].transform_keys(&:to_sym)) if include_metadata_in_message?
86
+ message[:timestamp] = (message[:timestamp_in_ms] / 1000).round if message.key?(:timestamp_in_ms) && !message.key?(:timestamp)
87
+ message[:datetime] = Time.at(message[:timestamp].to_i).to_datetime.to_s if message.key?(:timestamp)
81
88
  message
82
89
  end
83
90
 
@@ -94,8 +101,9 @@ module Legion
94
101
  require 'legion/extensions/tasker/runners/updater'
95
102
  sleep(delay_start)
96
103
  consumer_tag = "#{Legion::Settings[:client][:name]}_#{lex_name}_#{runner_name}_#{Thread.current.object_id}"
97
- @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag) do |delivery_info, metadata, payload|
98
- message = process_message(payload, metadata)
104
+ on_cancellation = block { cancel }
105
+ @consumer = @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag, on_cancellation: on_cancellation) do |delivery_info, metadata, payload|
106
+ message = process_message(payload, metadata, delivery_info)
99
107
  if use_runner?
100
108
  Legion::Runner.run(**message,
101
109
  runner_class: runner_class,
@@ -106,6 +114,8 @@ module Legion
106
114
  runner_class.send(find_function(message), **message)
107
115
  end
108
116
  @queue.acknowledge(delivery_info.delivery_tag) if manual_ack
117
+
118
+ cancel if Legion::Settings[:client][:shutting_down]
109
119
  rescue StandardError => e
110
120
  Legion::Logging.error e.message
111
121
  Legion::Logging.error e.backtrace
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -9,6 +7,7 @@ module Legion
9
7
  include Legion::Extensions::Builder::Base
10
8
 
11
9
  attr_reader :actors
10
+
12
11
  def build_actors
13
12
  @actors = {}
14
13
  require_files(actor_files)
@@ -19,7 +18,7 @@ module Legion
19
18
  def build_actor_list
20
19
  actor_files.each do |file|
21
20
  actor_name = file.split('/').last.sub('.rb', '')
22
- actor_class = lex_class.to_s + '::Actor::' + actor_name.split('_').collect(&:capitalize).join
21
+ actor_class = "#{lex_class}::Actor::#{actor_name.split('_').collect(&:capitalize).join}"
23
22
  @actors[actor_name.to_sym] = {
24
23
  extension: lex_class.to_s.downcase,
25
24
  extension_name: extension_name,
@@ -34,7 +33,7 @@ module Legion
34
33
  @runners.each do |runner, attr|
35
34
  next if @actors[runner.to_sym].is_a? Hash
36
35
 
37
- actor_class = attr[:extension_class].to_s + '::Actor::' + runner.to_s.split('_').collect(&:capitalize).join
36
+ actor_class = "#{attr[:extension_class]}::Actor::#{runner.to_s.split('_').collect(&:capitalize).join}"
38
37
  build_meta_actor(runner, attr) unless Kernel.const_defined? actor_class
39
38
  @actors[runner.to_sym] = {
40
39
  extension: attr[:extension],
@@ -49,7 +48,7 @@ module Legion
49
48
  def build_meta_actor(runner, attr)
50
49
  define_constant_two('Actor', root: lex_class)
51
50
 
52
- Kernel.const_get(attr[:extension_class].to_s + '::Actor')
51
+ Kernel.const_get("#{attr[:extension_class]}::Actor")
53
52
  .const_set(runner.to_s.split('_').collect(&:capitalize).join, Class.new(Legion::Extensions::Actors::Subscription))
54
53
  end
55
54
 
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  module Legion
4
2
  module Extensions
5
3
  module Builder
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -9,6 +7,7 @@ module Legion
9
7
  include Legion::Extensions::Builder::Base
10
8
 
11
9
  attr_reader :runners
10
+
12
11
  def build_runners
13
12
  @runners = {}
14
13
  lex_class.const_set('Runners', Module.new) unless lex_class.const_defined?('Runners')
@@ -19,7 +18,7 @@ module Legion
19
18
  def build_runner_list
20
19
  runner_files.each do |file|
21
20
  runner_name = file.split('/').last.sub('.rb', '')
22
- runner_class = lex_class.to_s + '::Runners::' + runner_name.split('_').collect(&:capitalize).join
21
+ runner_class = "#{lex_class}::Runners::#{runner_name.split('_').collect(&:capitalize).join}"
23
22
  loaded_runner = Kernel.const_get(runner_class)
24
23
  @runners[runner_name.to_sym] = {
25
24
  extension: lex_class.to_s.downcase,
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'builders/actors'
4
2
  require_relative 'builders/helpers'
5
3
  require_relative 'builders/runners'
@@ -45,11 +43,11 @@ module Legion
45
43
  end
46
44
 
47
45
  def build_transport
48
- if File.exist? extension_path + '/transport/autobuild.rb'
46
+ if File.exist? "#{extension_path}/transport/autobuild.rb"
49
47
  require "#{extension_path}/transport/autobuild"
50
48
  extension_class::Transport::AutoBuild.build
51
49
  log.warn 'still using transport::autobuild, please upgrade'
52
- elsif File.exist? extension_path + '/transport.rb'
50
+ elsif File.exist? "#{extension_path}/transport.rb"
53
51
  require "#{extension_path}/transport"
54
52
  extension_class::Transport.build
55
53
  else
@@ -58,7 +56,7 @@ module Legion
58
56
  end
59
57
  end
60
58
 
61
- def build_settings
59
+ def build_settings # rubocop:disable Metrics/AbcSize
62
60
  if Legion::Settings[:extensions].key?(lex_name.to_sym)
63
61
  Legion::Settings[:default_extension_settings].each do |key, value|
64
62
  Legion::Settings[:extensions][lex_name.to_sym][key.to_sym] = if Legion::Settings[:extensions][lex_name.to_sym].key?(key.to_sym)
@@ -70,6 +68,18 @@ module Legion
70
68
  else
71
69
  Legion::Settings[:extensions][lex_name.to_sym] = Legion::Settings[:default_extension_settings]
72
70
  end
71
+
72
+ default_settings.each do |key, value|
73
+ Legion::Settings[:extensions][lex_name.to_sym][key.to_sym] = if Legion::Settings[:extensions][lex_name.to_sym].key?(key.to_sym)
74
+ value.merge(Legion::Settings[:extensions][lex_name.to_sym][key.to_sym])
75
+ else
76
+ value
77
+ end
78
+ end
79
+ end
80
+
81
+ def default_settings
82
+ {}
73
83
  end
74
84
 
75
85
  def auto_generate_transport
@@ -0,0 +1,10 @@
1
+ module Legion
2
+ module Extensions
3
+ module Data
4
+ def build
5
+ @models = []
6
+ @migrations = []
7
+ end
8
+ end
9
+ end
10
+ end
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  module Legion
4
2
  module Extensions
5
3
  module Helpers
@@ -56,10 +54,6 @@ module Legion
56
54
  end
57
55
  alias extension_path full_path
58
56
 
59
- def to_json(object)
60
- Legion::JSON.dump(object)
61
- end
62
-
63
57
  def from_json(string)
64
58
  Legion::JSON.load(string)
65
59
  end
@@ -76,7 +70,7 @@ module Legion
76
70
  hash.each_with_object({}) do |(k, v), ret|
77
71
  key = recursive_key + k.to_s
78
72
  if v.is_a? Hash
79
- ret.merge! to_dotted_hash(v, key + '.')
73
+ ret.merge! to_dotted_hash(v, "#{key}.")
80
74
  else
81
75
  ret[key.to_sym] = v
82
76
  end
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
  module Legion
5
3
  module Extensions
@@ -11,8 +9,31 @@ module Legion
11
9
  if Legion::Settings[:extensions].key?(lex_filename.to_sym)
12
10
  Legion::Settings[:extensions][lex_filename.to_sym]
13
11
  else
14
- { logger: { level: 'info', extended: false } }
12
+ { logger: { level: 'info', extended: false, internal: false } }
13
+ end
14
+ end
15
+
16
+ # looks local, then in crypt, then settings, then cache, then env
17
+ def find_setting(name, **opts)
18
+ log.debug ".find_setting(#{name}) called"
19
+ return opts[name.to_sym] if opts.key? name.to_sym
20
+
21
+ string_name = "#{lex_name}_#{name.to_s.downcase}"
22
+ if Legion::Settings[:crypt][:vault][:connected] && Legion::Crypt.exist?(lex_name)
23
+ log.debug "looking for #{string_name} in Legion::Crypt"
24
+ crypt_result = Legion::Crypt.get(lex_name)
25
+ return crypt_result[name.to_sym] if crypt_result.is_a?(Hash) && crypt_result.key?(name.to_sym)
26
+ end
27
+ return settings[name.to_sym] if settings.key? name.to_sym
28
+
29
+ if Legion::Settings[:cache][:connected]
30
+ log.debug "looking for #{string_name} in Legion::Cache"
31
+ cache_result = Legion::Cache.get(string_name)
32
+ return cache_result unless cache_result.nil?
15
33
  end
34
+
35
+ ENV[string_name] if ENV.key? string_name
36
+ nil
16
37
  end
17
38
  end
18
39
  end
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  module Legion
4
2
  module Extensions
5
3
  module Helpers
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  module Legion
4
2
  module Extensions
5
3
  module Helpers
6
4
  module Logger
7
- def log
5
+ def log # rubocop:disable Metrics/AbcSize
8
6
  return @log unless @log.nil?
9
7
 
10
8
  logger_hash = { lex: lex_filename || nil }
@@ -12,6 +10,7 @@ module Legion
12
10
  if respond_to?(:settings) && settings.key?(:logger)
13
11
  logger_hash[:level] = settings[:logger][:level] if settings[:logger].key? :level
14
12
  logger_hash[:level] = settings[:logger][:log_file] if settings[:logger].key? :log_file
13
+ logger_hash[:level] = 'info' unless settings[:logger].key? :log_file
15
14
  logger_hash[:trace] = settings[:logger][:trace] if settings[:logger].key? :trace
16
15
  logger_hash[:extended] = settings[:logger][:extended] if settings[:logger].key? :extended
17
16
  elsif respond_to?(:settings)
@@ -20,7 +19,7 @@ module Legion
20
19
  else
21
20
  Legion::Logging.warn 'no settings'
22
21
  end
23
- @log = Legion::Logging::Logger.new(logger_hash)
22
+ @log = Legion::Logging::Logger.new(**logger_hash)
24
23
  end
25
24
 
26
25
  def handle_exception(exception, task_id: nil, **opts)
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require 'legion/transport'
4
2
  require 'legion/transport/messages/task_update'
5
3
  require 'legion/transport/messages/task_log'
@@ -19,7 +17,7 @@ module Legion
19
17
  %i[results payload function_args payload results].each do |column|
20
18
  update_hash[column] = opts[column] if opts.key? column
21
19
  end
22
- Legion::Transport::Messages::TaskUpdate.new(update_hash).publish
20
+ Legion::Transport::Messages::TaskUpdate.new(**update_hash).publish
23
21
  rescue StandardError => e
24
22
  log.fatal e.message
25
23
  log.fatal e.backtrace
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  require_relative 'base'
4
2
 
5
3
  module Legion
@@ -1,11 +1,10 @@
1
- # frozen_string_literal: true
2
-
3
1
  module Legion
4
2
  module Extensions
5
3
  module Transport
6
4
  include Legion::Extensions::Helpers::Transport
7
5
 
8
6
  attr_accessor :exchanges, :queues, :consumers, :messages
7
+
9
8
  def build
10
9
  @queues = []
11
10
  @exchanges = []
@@ -43,7 +42,7 @@ module Legion
43
42
  end
44
43
  end
45
44
 
46
- def auto_create_exchange(exchange, default_exchange = false)
45
+ def auto_create_exchange(exchange, default_exchange = false) # rubocop:disable Style/OptionalBooleanParameter
47
46
  if Object.const_defined? exchange
48
47
  Legion::Logging.warn "#{exchange} is already defined"
49
48
  return
@@ -63,7 +62,7 @@ module Legion
63
62
  return
64
63
  end
65
64
 
66
- Kernel.const_set(queue, Class.new(Legion::Transport::Queue))
65
+ transport_class::Queues.const_set(queue.split('::').last, Class.new(Legion::Transport::Queue))
67
66
  end
68
67
 
69
68
  def auto_create_dlx_exchange
@@ -82,7 +81,7 @@ module Legion
82
81
  return if transport_class::Queues.const_defined?('Dlx')
83
82
 
84
83
  special_name = default_exchange.new.exchange_name
85
- dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx"
84
+ dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx", auto_delete: false
86
85
  dlx_queue.bind("#{special_name}.dlx", { routing_key: '#' })
87
86
  end
88
87
 
@@ -1,9 +1,6 @@
1
- # frozen_string_literal: true
2
-
3
1
  require 'fileutils'
4
2
 
5
3
  module Legion
6
- # Responsible for starting the Legion process
7
4
  class Process
8
5
  def self.run!(options)
9
6
  Legion::Process.new(options).run!
@@ -51,7 +48,7 @@ module Legion
51
48
  trap_signals
52
49
 
53
50
  until quit
54
- sleep(1) # in real life, something productive would happen here
51
+ sleep(1)
55
52
  if @options.key? :time_limit
56
53
  @quit = true if Time.now - start_time > @options[:time_limit]
57
54
  end
@@ -113,16 +110,7 @@ module Legion
113
110
  :not_owned
114
111
  end
115
112
 
116
- #==========================================================================
117
- # SIGNAL HANDLING
118
- #==========================================================================
119
-
120
113
  def trap_signals
121
- # trap(:QUIT) do # graceful shutdown
122
- # info 'shutting down'
123
- # @quit = true
124
- # end
125
-
126
114
  trap('SIGTERM') do
127
115
  info 'sigterm'
128
116
  end