legionio 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +98 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +38 -8
  5. data/CHANGELOG.md +7 -0
  6. data/Gemfile +12 -9
  7. data/LICENSE.txt +21 -0
  8. data/README.md +46 -0
  9. data/Rakefile +1 -1
  10. data/bin/console +3 -2
  11. data/bin/legion +9 -6
  12. data/bin/test +28 -1
  13. data/bitbucket-pipelines.yml +13 -8
  14. data/legion.gemspec +27 -21
  15. data/lib/legion.rb +14 -5
  16. data/lib/legion/exceptions/handled_task.rb +6 -0
  17. data/lib/legion/exceptions/missingargument.rb +2 -2
  18. data/lib/legion/extensions.rb +151 -0
  19. data/lib/legion/extensions/actors/base.rb +53 -0
  20. data/lib/legion/extensions/actors/every.rb +50 -0
  21. data/lib/legion/extensions/actors/loop.rb +34 -0
  22. data/lib/legion/extensions/actors/nothing.rb +15 -0
  23. data/lib/legion/extensions/actors/once.rb +42 -0
  24. data/lib/legion/extensions/actors/poll.rb +90 -0
  25. data/lib/legion/extensions/actors/subscription.rb +120 -0
  26. data/lib/legion/extensions/builders/actors.rb +62 -0
  27. data/lib/legion/extensions/builders/base.rb +38 -0
  28. data/lib/legion/extensions/builders/helpers.rb +26 -0
  29. data/lib/legion/extensions/builders/runners.rb +54 -0
  30. data/lib/legion/extensions/core.rb +84 -0
  31. data/lib/legion/extensions/helpers/base.rb +88 -0
  32. data/lib/legion/extensions/helpers/core.rb +20 -0
  33. data/lib/legion/extensions/helpers/lex.rb +22 -0
  34. data/lib/legion/extensions/helpers/logger.rb +48 -0
  35. data/lib/legion/extensions/helpers/task.rb +42 -0
  36. data/lib/legion/extensions/helpers/transport.rb +45 -0
  37. data/lib/legion/extensions/transport.rb +156 -0
  38. data/lib/legion/process.rb +8 -3
  39. data/lib/legion/runner.rb +57 -0
  40. data/lib/legion/runner/log.rb +12 -0
  41. data/lib/legion/runner/status.rb +72 -0
  42. data/lib/legion/service.rb +44 -27
  43. data/lib/legion/supervison.rb +10 -24
  44. data/lib/legion/version.rb +1 -1
  45. metadata +184 -46
  46. data/lib/legion/extension/loader.rb +0 -96
  47. data/lib/legion/runners/runner.rb +0 -58
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+
5
+ module Legion
6
+ module Extensions
7
+ module Actors
8
+ class Subscription
9
+ include Concurrent::Async
10
+ include Legion::Extensions::Actors::Base
11
+ include Legion::Extensions::Helpers::Transport
12
+
13
+ def initialize(**_options)
14
+ super()
15
+ @queue = queue.new
16
+ rescue StandardError => e
17
+ log.fatal e.message
18
+ log.fatal e.backtrace
19
+ end
20
+
21
+ def create_queue
22
+ queues.const_set(actor_const, Class.new(Legion::Transport::Queue))
23
+ exchange_object = default_exchange.new
24
+ queue_object = Kernel.const_get(queue_string).new
25
+
26
+ queue_object.bind(exchange_object, routing_key: actor_name)
27
+ queue_object.bind(exchange_object, routing_key: "#{lex_name}.#{actor_name}.#")
28
+ end
29
+
30
+ def queue
31
+ create_queue unless queues.const_defined?(actor_const)
32
+ Kernel.const_get queue_string
33
+ end
34
+
35
+ def queue_string
36
+ @queue_string ||= "#{queues}::#{actor_const}"
37
+ end
38
+
39
+ def cancel
40
+ Legion::Logging.info "Closing subscription to #{@queue.name}"
41
+ @queue.channel.close
42
+ end
43
+
44
+ def block
45
+ false
46
+ end
47
+
48
+ def consumers
49
+ 1
50
+ end
51
+
52
+ def manual_ack
53
+ true
54
+ end
55
+
56
+ def delay_start
57
+ 0
58
+ end
59
+
60
+ def include_metadata_in_message?
61
+ true
62
+ end
63
+
64
+ def process_message(message, metadata)
65
+ payload = if metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/cs'
66
+ sleep(2) unless Legion::Settings[:crypt][:cs_encrypt_ready]
67
+ Legion::Crypt.decrypt(message)
68
+ elsif metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/pk'
69
+ Legion::Crypt.decrypt_from_keypair(metadata[:headers][:public_key], message)
70
+ else
71
+ message
72
+ end
73
+
74
+ message = if metadata[:content_type] == 'application/json'
75
+ Legion::JSON.load(payload)
76
+ else
77
+ payload
78
+ end
79
+
80
+ message = message.merge(metadata[:headers].transform_keys(&:to_sym)) if include_metadata_in_message?
81
+ message
82
+ end
83
+
84
+ def find_function(message = {})
85
+ return runner_function if actor_class.instance_methods(false).include?(:runner_function)
86
+ return function if actor_class.instance_methods(false).include?(:function)
87
+ return action if actor_class.instance_methods(false).include?(:action)
88
+ return message[:function] if message.key? :function
89
+
90
+ function
91
+ end
92
+
93
+ def subscribe
94
+ require 'legion/extensions/tasker/runners/updater'
95
+ sleep(delay_start)
96
+ consumer_tag = "#{Legion::Settings[:client][:name]}_#{lex_name}_#{runner_name}_#{Thread.current.object_id}"
97
+ @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag) do |delivery_info, metadata, payload|
98
+ message = process_message(payload, metadata)
99
+ if use_runner?
100
+ Legion::Runner.run(**message,
101
+ runner_class: runner_class,
102
+ function: find_function(message),
103
+ check_subtask: check_subtask?,
104
+ generate_task: generate_task?)
105
+ else
106
+ runner_class.send(find_function(message), **message)
107
+ end
108
+ @queue.acknowledge(delivery_info.delivery_tag) if manual_ack
109
+ rescue StandardError => e
110
+ Legion::Logging.error e.message
111
+ Legion::Logging.error e.backtrace
112
+ Legion::Logging.error message
113
+ Legion::Logging.error function
114
+ @queue.reject(delivery_info.delivery_tag) if manual_ack
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+
5
+ module Legion
6
+ module Extensions
7
+ module Builder
8
+ module Actors
9
+ include Legion::Extensions::Builder::Base
10
+
11
+ attr_reader :actors
12
+ def build_actors
13
+ @actors = {}
14
+ require_files(actor_files)
15
+ build_actor_list
16
+ build_meta_actor_list
17
+ end
18
+
19
+ def build_actor_list
20
+ actor_files.each do |file|
21
+ actor_name = file.split('/').last.sub('.rb', '')
22
+ actor_class = lex_class.to_s + '::Actor::' + actor_name.split('_').collect(&:capitalize).join
23
+ @actors[actor_name.to_sym] = {
24
+ extension: lex_class.to_s.downcase,
25
+ extension_name: extension_name,
26
+ actor_name: actor_name,
27
+ actor_class: Kernel.const_get(actor_class),
28
+ type: 'literal'
29
+ }
30
+ end
31
+ end
32
+
33
+ def build_meta_actor_list
34
+ @runners.each do |runner, attr|
35
+ next if @actors[runner.to_sym].is_a? Hash
36
+
37
+ actor_class = attr[:extension_class].to_s + '::Actor::' + runner.to_s.split('_').collect(&:capitalize).join
38
+ build_meta_actor(runner, attr) unless Kernel.const_defined? actor_class
39
+ @actors[runner.to_sym] = {
40
+ extension: attr[:extension],
41
+ extension_name: attr[:extension_name],
42
+ actor_name: attr[:runner_name],
43
+ actor_class: Kernel.const_get(actor_class),
44
+ type: 'meta'
45
+ }
46
+ end
47
+ end
48
+
49
+ def build_meta_actor(runner, attr)
50
+ define_constant_two('Actor', root: lex_class)
51
+
52
+ Kernel.const_get(attr[:extension_class].to_s + '::Actor')
53
+ .const_set(runner.to_s.split('_').collect(&:capitalize).join, Class.new(Legion::Extensions::Actors::Subscription))
54
+ end
55
+
56
+ def actor_files
57
+ @actor_files ||= find_files('actors')
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Builder
6
+ module Base
7
+ def find_files(name, path = extension_path)
8
+ files = []
9
+ return files unless Dir.exist? "#{path}/#{name}"
10
+
11
+ Dir["#{path}/#{name}/*.rb"].each do |file|
12
+ files.push(file)
13
+ end
14
+ files
15
+ end
16
+
17
+ def require_files(files)
18
+ files.each { |file| require file }
19
+ end
20
+
21
+ def const_defined_two?(item, root = Kernel)
22
+ root.const_defined?(item.to_s)
23
+ end
24
+
25
+ def define_constant_two(item, root: Kernel, type: Module)
26
+ return true if const_defined?(item, root: root)
27
+
28
+ root.const_set(item.to_s, type.new)
29
+ end
30
+
31
+ def define_get(item, root: Kernel, type: Module)
32
+ define_constant_two(item, root: root, type: type) if const_defined_two?(item, root: root)
33
+ root.const_get(item)
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+
5
+ module Legion
6
+ module Extensions
7
+ module Builder
8
+ module Helpers
9
+ include Legion::Extensions::Builder::Base
10
+
11
+ def build_helpers
12
+ @helpers ||= []
13
+ @helpers.push(require_files(helper_files))
14
+ end
15
+
16
+ def helper_files
17
+ @helper_files ||= find_files('helpers')
18
+ end
19
+
20
+ def helpers
21
+ @helpers
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+
5
+ module Legion
6
+ module Extensions
7
+ module Builder
8
+ module Runners
9
+ include Legion::Extensions::Builder::Base
10
+
11
+ attr_reader :runners
12
+ def build_runners
13
+ @runners = {}
14
+ lex_class.const_set('Runners', Module.new) unless lex_class.const_defined?('Runners')
15
+ require_files(runner_files)
16
+ build_runner_list
17
+ end
18
+
19
+ def build_runner_list
20
+ runner_files.each do |file|
21
+ runner_name = file.split('/').last.sub('.rb', '')
22
+ runner_class = lex_class.to_s + '::Runners::' + runner_name.split('_').collect(&:capitalize).join
23
+ loaded_runner = Kernel.const_get(runner_class)
24
+ @runners[runner_name.to_sym] = {
25
+ extension: lex_class.to_s.downcase,
26
+ extension_name: extension_name,
27
+ extension_class: lex_class,
28
+ runner_name: runner_name,
29
+ runner_class: runner_class,
30
+ runner_path: file,
31
+ class_methods: {}
32
+ }
33
+
34
+ loaded_runner.public_instance_methods(false).each do |runner_method|
35
+ @runners[runner_name.to_sym][:class_methods][runner_method] = {
36
+ args: loaded_runner.instance_method(runner_method).parameters
37
+ }
38
+ end
39
+
40
+ loaded_runner.methods(false).each do |runner_method|
41
+ @runners[runner_name.to_sym][:class_methods][runner_method] = {
42
+ args: loaded_runner.method(runner_method).parameters
43
+ }
44
+ end
45
+ end
46
+ end
47
+
48
+ def runner_files
49
+ @runner_files ||= find_files('runners')
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'builders/actors'
4
+ require_relative 'builders/helpers'
5
+ require_relative 'builders/runners'
6
+
7
+ require_relative 'helpers/core'
8
+ require_relative 'helpers/task'
9
+ require_relative 'helpers/logger'
10
+ require_relative 'helpers/lex'
11
+ require_relative 'helpers/transport'
12
+
13
+ require_relative 'actors/base'
14
+ require_relative 'actors/every'
15
+ require_relative 'actors/loop'
16
+ require_relative 'actors/once'
17
+ require_relative 'actors/poll'
18
+ require_relative 'actors/subscription'
19
+ require_relative 'actors/nothing'
20
+
21
+ module Legion
22
+ module Extensions
23
+ module Core
24
+ include Legion::Extensions::Helpers::Transport
25
+ include Legion::Extensions::Helpers::Lex
26
+
27
+ include Legion::Extensions::Builder::Runners
28
+ include Legion::Extensions::Builder::Helpers
29
+ include Legion::Extensions::Builder::Actors
30
+
31
+ def autobuild
32
+ @actors = {}
33
+ @meta_actors = {}
34
+ @runners = {}
35
+ @helpers = []
36
+
37
+ @queues = {}
38
+ @exchanges = {}
39
+ @messages = {}
40
+ build_settings
41
+ build_transport
42
+ build_helpers
43
+ build_runners
44
+ build_actors
45
+ end
46
+
47
+ def build_transport
48
+ if File.exist? extension_path + '/transport/autobuild.rb'
49
+ require "#{extension_path}/transport/autobuild"
50
+ extension_class::Transport::AutoBuild.build
51
+ log.warn 'still using transport::autobuild, please upgrade'
52
+ elsif File.exist? extension_path + '/transport.rb'
53
+ require "#{extension_path}/transport"
54
+ extension_class::Transport.build
55
+ else
56
+ auto_generate_transport
57
+ extension_class::Transport.build
58
+ end
59
+ end
60
+
61
+ def build_settings
62
+ if Legion::Settings[:extensions].key?(lex_name.to_sym)
63
+ Legion::Settings[:default_extension_settings].each do |key, value|
64
+ Legion::Settings[:extensions][lex_name.to_sym][key.to_sym] = if Legion::Settings[:extensions][lex_name.to_sym].key?(key.to_sym)
65
+ value.merge(Legion::Settings[:extensions][lex_name.to_sym][key.to_sym])
66
+ else
67
+ value
68
+ end
69
+ end
70
+ else
71
+ Legion::Settings[:extensions][lex_name.to_sym] = Legion::Settings[:default_extension_settings]
72
+ end
73
+ end
74
+
75
+ def auto_generate_transport
76
+ require 'legion/extensions/transport'
77
+ log.debug 'running meta magic to generate a transport base class'
78
+ return if Kernel.const_defined? "#{lex_class}::Transport"
79
+
80
+ Kernel.const_get(lex_class.to_s).const_set('Transport', Module.new { extend Legion::Extensions::Transport })
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Helpers
6
+ module Base
7
+ def lex_class
8
+ @lex_class ||= Kernel.const_get(calling_class_array[0..2].join('::'))
9
+ end
10
+ alias extension_class lex_class
11
+
12
+ def lex_name
13
+ @lex_name ||= calling_class_array[2].gsub(/(?<!^)[A-Z]/) { "_#{$&}" }.downcase
14
+ end
15
+ alias extension_name lex_name
16
+ alias lex_filename lex_name
17
+
18
+ def lex_const
19
+ @lex_const ||= calling_class_array[2]
20
+ end
21
+
22
+ def calling_class
23
+ @calling_class ||= respond_to?(:ancestors) ? ancestors.first : self.class
24
+ end
25
+
26
+ def calling_class_array
27
+ @calling_class_array ||= calling_class.to_s.split('::')
28
+ end
29
+
30
+ def actor_class
31
+ calling_class
32
+ end
33
+
34
+ def actor_name
35
+ @actor_name ||= calling_class_array.last.gsub(/(?<!^)[A-Z]/) { "_#{$&}" }.downcase
36
+ end
37
+
38
+ def actor_const
39
+ @actor_const ||= calling_class_array.last
40
+ end
41
+
42
+ def runner_class
43
+ @runner_class ||= Kernel.const_get(actor_class.to_s.sub!('Actor', 'Runners'))
44
+ end
45
+
46
+ def runner_name
47
+ @runner_name ||= runner_class.to_s.split('::').last.gsub(/(?<!^)[A-Z]/) { "_#{$&}" }.downcase
48
+ end
49
+
50
+ def runner_const
51
+ @runner_const ||= runner_class.to_s.split('::').last
52
+ end
53
+
54
+ def full_path
55
+ @full_path ||= "#{Gem::Specification.find_by_name("lex-#{lex_name}").gem_dir}/lib/legion/extensions/#{lex_filename}"
56
+ end
57
+ alias extension_path full_path
58
+
59
+ def to_json(object)
60
+ Legion::JSON.dump(object)
61
+ end
62
+
63
+ def from_json(string)
64
+ Legion::JSON.load(string)
65
+ end
66
+
67
+ def normalize(thing)
68
+ if thing.is_a? String
69
+ to_json(from_json(thing))
70
+ else
71
+ from_json(to_json(thing))
72
+ end
73
+ end
74
+
75
+ def to_dotted_hash(hash, recursive_key = '')
76
+ hash.each_with_object({}) do |(k, v), ret|
77
+ key = recursive_key + k.to_s
78
+ if v.is_a? Hash
79
+ ret.merge! to_dotted_hash(v, key + '.')
80
+ else
81
+ ret[key.to_sym] = v
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end