legionio 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.circleci/config.yml +98 -0
- data/.rspec +2 -0
- data/.rubocop.yml +38 -8
- data/CHANGELOG.md +7 -0
- data/Gemfile +12 -9
- data/LICENSE.txt +21 -0
- data/README.md +46 -0
- data/Rakefile +1 -1
- data/bin/console +3 -2
- data/bin/legion +9 -6
- data/bin/test +28 -1
- data/bitbucket-pipelines.yml +13 -8
- data/legion.gemspec +27 -21
- data/lib/legion.rb +14 -5
- data/lib/legion/exceptions/handled_task.rb +6 -0
- data/lib/legion/exceptions/missingargument.rb +2 -2
- data/lib/legion/extensions.rb +151 -0
- data/lib/legion/extensions/actors/base.rb +53 -0
- data/lib/legion/extensions/actors/every.rb +50 -0
- data/lib/legion/extensions/actors/loop.rb +34 -0
- data/lib/legion/extensions/actors/nothing.rb +15 -0
- data/lib/legion/extensions/actors/once.rb +42 -0
- data/lib/legion/extensions/actors/poll.rb +90 -0
- data/lib/legion/extensions/actors/subscription.rb +120 -0
- data/lib/legion/extensions/builders/actors.rb +62 -0
- data/lib/legion/extensions/builders/base.rb +38 -0
- data/lib/legion/extensions/builders/helpers.rb +26 -0
- data/lib/legion/extensions/builders/runners.rb +54 -0
- data/lib/legion/extensions/core.rb +84 -0
- data/lib/legion/extensions/helpers/base.rb +88 -0
- data/lib/legion/extensions/helpers/core.rb +20 -0
- data/lib/legion/extensions/helpers/lex.rb +22 -0
- data/lib/legion/extensions/helpers/logger.rb +48 -0
- data/lib/legion/extensions/helpers/task.rb +42 -0
- data/lib/legion/extensions/helpers/transport.rb +45 -0
- data/lib/legion/extensions/transport.rb +156 -0
- data/lib/legion/process.rb +8 -3
- data/lib/legion/runner.rb +57 -0
- data/lib/legion/runner/log.rb +12 -0
- data/lib/legion/runner/status.rb +72 -0
- data/lib/legion/service.rb +44 -27
- data/lib/legion/supervison.rb +10 -24
- data/lib/legion/version.rb +1 -1
- metadata +184 -46
- data/lib/legion/extension/loader.rb +0 -96
- data/lib/legion/runners/runner.rb +0 -58
@@ -0,0 +1,120 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
|
5
|
+
module Legion
|
6
|
+
module Extensions
|
7
|
+
module Actors
|
8
|
+
class Subscription
|
9
|
+
include Concurrent::Async
|
10
|
+
include Legion::Extensions::Actors::Base
|
11
|
+
include Legion::Extensions::Helpers::Transport
|
12
|
+
|
13
|
+
def initialize(**_options)
|
14
|
+
super()
|
15
|
+
@queue = queue.new
|
16
|
+
rescue StandardError => e
|
17
|
+
log.fatal e.message
|
18
|
+
log.fatal e.backtrace
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_queue
|
22
|
+
queues.const_set(actor_const, Class.new(Legion::Transport::Queue))
|
23
|
+
exchange_object = default_exchange.new
|
24
|
+
queue_object = Kernel.const_get(queue_string).new
|
25
|
+
|
26
|
+
queue_object.bind(exchange_object, routing_key: actor_name)
|
27
|
+
queue_object.bind(exchange_object, routing_key: "#{lex_name}.#{actor_name}.#")
|
28
|
+
end
|
29
|
+
|
30
|
+
def queue
|
31
|
+
create_queue unless queues.const_defined?(actor_const)
|
32
|
+
Kernel.const_get queue_string
|
33
|
+
end
|
34
|
+
|
35
|
+
def queue_string
|
36
|
+
@queue_string ||= "#{queues}::#{actor_const}"
|
37
|
+
end
|
38
|
+
|
39
|
+
def cancel
|
40
|
+
Legion::Logging.info "Closing subscription to #{@queue.name}"
|
41
|
+
@queue.channel.close
|
42
|
+
end
|
43
|
+
|
44
|
+
def block
|
45
|
+
false
|
46
|
+
end
|
47
|
+
|
48
|
+
def consumers
|
49
|
+
1
|
50
|
+
end
|
51
|
+
|
52
|
+
def manual_ack
|
53
|
+
true
|
54
|
+
end
|
55
|
+
|
56
|
+
def delay_start
|
57
|
+
0
|
58
|
+
end
|
59
|
+
|
60
|
+
def include_metadata_in_message?
|
61
|
+
true
|
62
|
+
end
|
63
|
+
|
64
|
+
def process_message(message, metadata)
|
65
|
+
payload = if metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/cs'
|
66
|
+
sleep(2) unless Legion::Settings[:crypt][:cs_encrypt_ready]
|
67
|
+
Legion::Crypt.decrypt(message)
|
68
|
+
elsif metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/pk'
|
69
|
+
Legion::Crypt.decrypt_from_keypair(metadata[:headers][:public_key], message)
|
70
|
+
else
|
71
|
+
message
|
72
|
+
end
|
73
|
+
|
74
|
+
message = if metadata[:content_type] == 'application/json'
|
75
|
+
Legion::JSON.load(payload)
|
76
|
+
else
|
77
|
+
payload
|
78
|
+
end
|
79
|
+
|
80
|
+
message = message.merge(metadata[:headers].transform_keys(&:to_sym)) if include_metadata_in_message?
|
81
|
+
message
|
82
|
+
end
|
83
|
+
|
84
|
+
def find_function(message = {})
|
85
|
+
return runner_function if actor_class.instance_methods(false).include?(:runner_function)
|
86
|
+
return function if actor_class.instance_methods(false).include?(:function)
|
87
|
+
return action if actor_class.instance_methods(false).include?(:action)
|
88
|
+
return message[:function] if message.key? :function
|
89
|
+
|
90
|
+
function
|
91
|
+
end
|
92
|
+
|
93
|
+
def subscribe
|
94
|
+
require 'legion/extensions/tasker/runners/updater'
|
95
|
+
sleep(delay_start)
|
96
|
+
consumer_tag = "#{Legion::Settings[:client][:name]}_#{lex_name}_#{runner_name}_#{Thread.current.object_id}"
|
97
|
+
@queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag) do |delivery_info, metadata, payload|
|
98
|
+
message = process_message(payload, metadata)
|
99
|
+
if use_runner?
|
100
|
+
Legion::Runner.run(**message,
|
101
|
+
runner_class: runner_class,
|
102
|
+
function: find_function(message),
|
103
|
+
check_subtask: check_subtask?,
|
104
|
+
generate_task: generate_task?)
|
105
|
+
else
|
106
|
+
runner_class.send(find_function(message), **message)
|
107
|
+
end
|
108
|
+
@queue.acknowledge(delivery_info.delivery_tag) if manual_ack
|
109
|
+
rescue StandardError => e
|
110
|
+
Legion::Logging.error e.message
|
111
|
+
Legion::Logging.error e.backtrace
|
112
|
+
Legion::Logging.error message
|
113
|
+
Legion::Logging.error function
|
114
|
+
@queue.reject(delivery_info.delivery_tag) if manual_ack
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
|
5
|
+
module Legion
|
6
|
+
module Extensions
|
7
|
+
module Builder
|
8
|
+
module Actors
|
9
|
+
include Legion::Extensions::Builder::Base
|
10
|
+
|
11
|
+
attr_reader :actors
|
12
|
+
def build_actors
|
13
|
+
@actors = {}
|
14
|
+
require_files(actor_files)
|
15
|
+
build_actor_list
|
16
|
+
build_meta_actor_list
|
17
|
+
end
|
18
|
+
|
19
|
+
def build_actor_list
|
20
|
+
actor_files.each do |file|
|
21
|
+
actor_name = file.split('/').last.sub('.rb', '')
|
22
|
+
actor_class = lex_class.to_s + '::Actor::' + actor_name.split('_').collect(&:capitalize).join
|
23
|
+
@actors[actor_name.to_sym] = {
|
24
|
+
extension: lex_class.to_s.downcase,
|
25
|
+
extension_name: extension_name,
|
26
|
+
actor_name: actor_name,
|
27
|
+
actor_class: Kernel.const_get(actor_class),
|
28
|
+
type: 'literal'
|
29
|
+
}
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def build_meta_actor_list
|
34
|
+
@runners.each do |runner, attr|
|
35
|
+
next if @actors[runner.to_sym].is_a? Hash
|
36
|
+
|
37
|
+
actor_class = attr[:extension_class].to_s + '::Actor::' + runner.to_s.split('_').collect(&:capitalize).join
|
38
|
+
build_meta_actor(runner, attr) unless Kernel.const_defined? actor_class
|
39
|
+
@actors[runner.to_sym] = {
|
40
|
+
extension: attr[:extension],
|
41
|
+
extension_name: attr[:extension_name],
|
42
|
+
actor_name: attr[:runner_name],
|
43
|
+
actor_class: Kernel.const_get(actor_class),
|
44
|
+
type: 'meta'
|
45
|
+
}
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def build_meta_actor(runner, attr)
|
50
|
+
define_constant_two('Actor', root: lex_class)
|
51
|
+
|
52
|
+
Kernel.const_get(attr[:extension_class].to_s + '::Actor')
|
53
|
+
.const_set(runner.to_s.split('_').collect(&:capitalize).join, Class.new(Legion::Extensions::Actors::Subscription))
|
54
|
+
end
|
55
|
+
|
56
|
+
def actor_files
|
57
|
+
@actor_files ||= find_files('actors')
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Extensions
|
5
|
+
module Builder
|
6
|
+
module Base
|
7
|
+
def find_files(name, path = extension_path)
|
8
|
+
files = []
|
9
|
+
return files unless Dir.exist? "#{path}/#{name}"
|
10
|
+
|
11
|
+
Dir["#{path}/#{name}/*.rb"].each do |file|
|
12
|
+
files.push(file)
|
13
|
+
end
|
14
|
+
files
|
15
|
+
end
|
16
|
+
|
17
|
+
def require_files(files)
|
18
|
+
files.each { |file| require file }
|
19
|
+
end
|
20
|
+
|
21
|
+
def const_defined_two?(item, root = Kernel)
|
22
|
+
root.const_defined?(item.to_s)
|
23
|
+
end
|
24
|
+
|
25
|
+
def define_constant_two(item, root: Kernel, type: Module)
|
26
|
+
return true if const_defined?(item, root: root)
|
27
|
+
|
28
|
+
root.const_set(item.to_s, type.new)
|
29
|
+
end
|
30
|
+
|
31
|
+
def define_get(item, root: Kernel, type: Module)
|
32
|
+
define_constant_two(item, root: root, type: type) if const_defined_two?(item, root: root)
|
33
|
+
root.const_get(item)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
|
5
|
+
module Legion
|
6
|
+
module Extensions
|
7
|
+
module Builder
|
8
|
+
module Helpers
|
9
|
+
include Legion::Extensions::Builder::Base
|
10
|
+
|
11
|
+
def build_helpers
|
12
|
+
@helpers ||= []
|
13
|
+
@helpers.push(require_files(helper_files))
|
14
|
+
end
|
15
|
+
|
16
|
+
def helper_files
|
17
|
+
@helper_files ||= find_files('helpers')
|
18
|
+
end
|
19
|
+
|
20
|
+
def helpers
|
21
|
+
@helpers
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
|
5
|
+
module Legion
|
6
|
+
module Extensions
|
7
|
+
module Builder
|
8
|
+
module Runners
|
9
|
+
include Legion::Extensions::Builder::Base
|
10
|
+
|
11
|
+
attr_reader :runners
|
12
|
+
def build_runners
|
13
|
+
@runners = {}
|
14
|
+
lex_class.const_set('Runners', Module.new) unless lex_class.const_defined?('Runners')
|
15
|
+
require_files(runner_files)
|
16
|
+
build_runner_list
|
17
|
+
end
|
18
|
+
|
19
|
+
def build_runner_list
|
20
|
+
runner_files.each do |file|
|
21
|
+
runner_name = file.split('/').last.sub('.rb', '')
|
22
|
+
runner_class = lex_class.to_s + '::Runners::' + runner_name.split('_').collect(&:capitalize).join
|
23
|
+
loaded_runner = Kernel.const_get(runner_class)
|
24
|
+
@runners[runner_name.to_sym] = {
|
25
|
+
extension: lex_class.to_s.downcase,
|
26
|
+
extension_name: extension_name,
|
27
|
+
extension_class: lex_class,
|
28
|
+
runner_name: runner_name,
|
29
|
+
runner_class: runner_class,
|
30
|
+
runner_path: file,
|
31
|
+
class_methods: {}
|
32
|
+
}
|
33
|
+
|
34
|
+
loaded_runner.public_instance_methods(false).each do |runner_method|
|
35
|
+
@runners[runner_name.to_sym][:class_methods][runner_method] = {
|
36
|
+
args: loaded_runner.instance_method(runner_method).parameters
|
37
|
+
}
|
38
|
+
end
|
39
|
+
|
40
|
+
loaded_runner.methods(false).each do |runner_method|
|
41
|
+
@runners[runner_name.to_sym][:class_methods][runner_method] = {
|
42
|
+
args: loaded_runner.method(runner_method).parameters
|
43
|
+
}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def runner_files
|
49
|
+
@runner_files ||= find_files('runners')
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'builders/actors'
|
4
|
+
require_relative 'builders/helpers'
|
5
|
+
require_relative 'builders/runners'
|
6
|
+
|
7
|
+
require_relative 'helpers/core'
|
8
|
+
require_relative 'helpers/task'
|
9
|
+
require_relative 'helpers/logger'
|
10
|
+
require_relative 'helpers/lex'
|
11
|
+
require_relative 'helpers/transport'
|
12
|
+
|
13
|
+
require_relative 'actors/base'
|
14
|
+
require_relative 'actors/every'
|
15
|
+
require_relative 'actors/loop'
|
16
|
+
require_relative 'actors/once'
|
17
|
+
require_relative 'actors/poll'
|
18
|
+
require_relative 'actors/subscription'
|
19
|
+
require_relative 'actors/nothing'
|
20
|
+
|
21
|
+
module Legion
|
22
|
+
module Extensions
|
23
|
+
module Core
|
24
|
+
include Legion::Extensions::Helpers::Transport
|
25
|
+
include Legion::Extensions::Helpers::Lex
|
26
|
+
|
27
|
+
include Legion::Extensions::Builder::Runners
|
28
|
+
include Legion::Extensions::Builder::Helpers
|
29
|
+
include Legion::Extensions::Builder::Actors
|
30
|
+
|
31
|
+
def autobuild
|
32
|
+
@actors = {}
|
33
|
+
@meta_actors = {}
|
34
|
+
@runners = {}
|
35
|
+
@helpers = []
|
36
|
+
|
37
|
+
@queues = {}
|
38
|
+
@exchanges = {}
|
39
|
+
@messages = {}
|
40
|
+
build_settings
|
41
|
+
build_transport
|
42
|
+
build_helpers
|
43
|
+
build_runners
|
44
|
+
build_actors
|
45
|
+
end
|
46
|
+
|
47
|
+
def build_transport
|
48
|
+
if File.exist? extension_path + '/transport/autobuild.rb'
|
49
|
+
require "#{extension_path}/transport/autobuild"
|
50
|
+
extension_class::Transport::AutoBuild.build
|
51
|
+
log.warn 'still using transport::autobuild, please upgrade'
|
52
|
+
elsif File.exist? extension_path + '/transport.rb'
|
53
|
+
require "#{extension_path}/transport"
|
54
|
+
extension_class::Transport.build
|
55
|
+
else
|
56
|
+
auto_generate_transport
|
57
|
+
extension_class::Transport.build
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def build_settings
|
62
|
+
if Legion::Settings[:extensions].key?(lex_name.to_sym)
|
63
|
+
Legion::Settings[:default_extension_settings].each do |key, value|
|
64
|
+
Legion::Settings[:extensions][lex_name.to_sym][key.to_sym] = if Legion::Settings[:extensions][lex_name.to_sym].key?(key.to_sym)
|
65
|
+
value.merge(Legion::Settings[:extensions][lex_name.to_sym][key.to_sym])
|
66
|
+
else
|
67
|
+
value
|
68
|
+
end
|
69
|
+
end
|
70
|
+
else
|
71
|
+
Legion::Settings[:extensions][lex_name.to_sym] = Legion::Settings[:default_extension_settings]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def auto_generate_transport
|
76
|
+
require 'legion/extensions/transport'
|
77
|
+
log.debug 'running meta magic to generate a transport base class'
|
78
|
+
return if Kernel.const_defined? "#{lex_class}::Transport"
|
79
|
+
|
80
|
+
Kernel.const_get(lex_class.to_s).const_set('Transport', Module.new { extend Legion::Extensions::Transport })
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Legion
|
4
|
+
module Extensions
|
5
|
+
module Helpers
|
6
|
+
module Base
|
7
|
+
def lex_class
|
8
|
+
@lex_class ||= Kernel.const_get(calling_class_array[0..2].join('::'))
|
9
|
+
end
|
10
|
+
alias extension_class lex_class
|
11
|
+
|
12
|
+
def lex_name
|
13
|
+
@lex_name ||= calling_class_array[2].gsub(/(?<!^)[A-Z]/) { "_#{$&}" }.downcase
|
14
|
+
end
|
15
|
+
alias extension_name lex_name
|
16
|
+
alias lex_filename lex_name
|
17
|
+
|
18
|
+
def lex_const
|
19
|
+
@lex_const ||= calling_class_array[2]
|
20
|
+
end
|
21
|
+
|
22
|
+
def calling_class
|
23
|
+
@calling_class ||= respond_to?(:ancestors) ? ancestors.first : self.class
|
24
|
+
end
|
25
|
+
|
26
|
+
def calling_class_array
|
27
|
+
@calling_class_array ||= calling_class.to_s.split('::')
|
28
|
+
end
|
29
|
+
|
30
|
+
def actor_class
|
31
|
+
calling_class
|
32
|
+
end
|
33
|
+
|
34
|
+
def actor_name
|
35
|
+
@actor_name ||= calling_class_array.last.gsub(/(?<!^)[A-Z]/) { "_#{$&}" }.downcase
|
36
|
+
end
|
37
|
+
|
38
|
+
def actor_const
|
39
|
+
@actor_const ||= calling_class_array.last
|
40
|
+
end
|
41
|
+
|
42
|
+
def runner_class
|
43
|
+
@runner_class ||= Kernel.const_get(actor_class.to_s.sub!('Actor', 'Runners'))
|
44
|
+
end
|
45
|
+
|
46
|
+
def runner_name
|
47
|
+
@runner_name ||= runner_class.to_s.split('::').last.gsub(/(?<!^)[A-Z]/) { "_#{$&}" }.downcase
|
48
|
+
end
|
49
|
+
|
50
|
+
def runner_const
|
51
|
+
@runner_const ||= runner_class.to_s.split('::').last
|
52
|
+
end
|
53
|
+
|
54
|
+
def full_path
|
55
|
+
@full_path ||= "#{Gem::Specification.find_by_name("lex-#{lex_name}").gem_dir}/lib/legion/extensions/#{lex_filename}"
|
56
|
+
end
|
57
|
+
alias extension_path full_path
|
58
|
+
|
59
|
+
def to_json(object)
|
60
|
+
Legion::JSON.dump(object)
|
61
|
+
end
|
62
|
+
|
63
|
+
def from_json(string)
|
64
|
+
Legion::JSON.load(string)
|
65
|
+
end
|
66
|
+
|
67
|
+
def normalize(thing)
|
68
|
+
if thing.is_a? String
|
69
|
+
to_json(from_json(thing))
|
70
|
+
else
|
71
|
+
from_json(to_json(thing))
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def to_dotted_hash(hash, recursive_key = '')
|
76
|
+
hash.each_with_object({}) do |(k, v), ret|
|
77
|
+
key = recursive_key + k.to_s
|
78
|
+
if v.is_a? Hash
|
79
|
+
ret.merge! to_dotted_hash(v, key + '.')
|
80
|
+
else
|
81
|
+
ret[key.to_sym] = v
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|