legionio 0.3.5 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +136 -0
  3. data/.gitignore +15 -0
  4. data/.rspec +2 -0
  5. data/.rubocop.yml +94 -0
  6. data/CHANGELOG.md +37 -0
  7. data/Dockerfile +9 -0
  8. data/Gemfile +3 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +162 -0
  11. data/Rakefile +32 -0
  12. data/bitbucket-pipelines.yml +20 -0
  13. data/docker_deploy.rb +8 -0
  14. data/exe/legion +4 -0
  15. data/exe/legionio +53 -0
  16. data/exe/lex_gen +5 -0
  17. data/legionio.gemspec +64 -0
  18. data/lib/legion.rb +25 -0
  19. data/lib/legion/cli.rb +56 -0
  20. data/lib/legion/cli/chain.rb +35 -0
  21. data/lib/legion/cli/cohort.rb +10 -0
  22. data/lib/legion/cli/function.rb +41 -0
  23. data/lib/legion/cli/lex/actor.rb +31 -0
  24. data/lib/legion/cli/lex/exchange.rb +32 -0
  25. data/lib/legion/cli/lex/message.rb +32 -0
  26. data/lib/legion/cli/lex/queue.rb +45 -0
  27. data/lib/legion/cli/lex/runner.rb +70 -0
  28. data/lib/legion/cli/lex/templates/actor.erb +6 -0
  29. data/lib/legion/cli/lex/templates/actor_spec.erb +0 -0
  30. data/lib/legion/cli/lex/templates/base/bitbucket.yml.erb +69 -0
  31. data/lib/legion/cli/lex/templates/base/gemfile.erb +3 -0
  32. data/lib/legion/cli/lex/templates/base/gemspec.erb +26 -0
  33. data/lib/legion/cli/lex/templates/base/gitignore.erb +11 -0
  34. data/lib/legion/cli/lex/templates/base/lex.erb +9 -0
  35. data/lib/legion/cli/lex/templates/base/lex_spec.erb +5 -0
  36. data/lib/legion/cli/lex/templates/base/lic.erb +21 -0
  37. data/lib/legion/cli/lex/templates/base/rakefile.erb +6 -0
  38. data/lib/legion/cli/lex/templates/base/readme.md.erb +2 -0
  39. data/lib/legion/cli/lex/templates/base/rubocop.yml.erb +15 -0
  40. data/lib/legion/cli/lex/templates/base/spec_helper.rb.erb +11 -0
  41. data/lib/legion/cli/lex/templates/base/version.erb +7 -0
  42. data/lib/legion/cli/lex/templates/exchange.erb +11 -0
  43. data/lib/legion/cli/lex/templates/exchange_spec.erb +0 -0
  44. data/lib/legion/cli/lex/templates/message.erb +23 -0
  45. data/lib/legion/cli/lex/templates/message_spec.erb +0 -0
  46. data/lib/legion/cli/lex/templates/queue.erb +12 -0
  47. data/lib/legion/cli/lex/templates/queue_helper.erb +24 -0
  48. data/lib/legion/cli/lex/templates/queue_spec.erb +11 -0
  49. data/lib/legion/cli/lex/templates/runner.erb +11 -0
  50. data/lib/legion/cli/lex/templates/runner_spec.erb +11 -0
  51. data/lib/legion/cli/relationship.rb +22 -0
  52. data/lib/legion/cli/task.rb +49 -0
  53. data/lib/legion/cli/trigger.rb +88 -0
  54. data/lib/legion/cli/version.rb +5 -0
  55. data/lib/legion/extensions.rb +219 -0
  56. data/lib/legion/extensions/actors/base.rb +47 -0
  57. data/lib/legion/extensions/actors/every.rb +48 -0
  58. data/lib/legion/extensions/actors/loop.rb +32 -0
  59. data/lib/legion/extensions/actors/nothing.rb +15 -0
  60. data/lib/legion/extensions/actors/once.rb +40 -0
  61. data/lib/legion/extensions/actors/poll.rb +87 -0
  62. data/lib/legion/extensions/actors/subscription.rb +139 -0
  63. data/lib/legion/extensions/builders/actors.rb +61 -0
  64. data/lib/legion/extensions/builders/base.rb +36 -0
  65. data/lib/legion/extensions/builders/helpers.rb +24 -0
  66. data/lib/legion/extensions/builders/runners.rb +58 -0
  67. data/lib/legion/extensions/core.rb +131 -0
  68. data/lib/legion/extensions/data.rb +58 -0
  69. data/lib/legion/extensions/data/migrator.rb +28 -0
  70. data/lib/legion/extensions/data/model.rb +8 -0
  71. data/lib/legion/extensions/helpers/base.rb +82 -0
  72. data/lib/legion/extensions/helpers/cache.rb +23 -0
  73. data/lib/legion/extensions/helpers/core.rb +41 -0
  74. data/lib/legion/extensions/helpers/data.rb +23 -0
  75. data/lib/legion/extensions/helpers/lex.rb +48 -0
  76. data/lib/legion/extensions/helpers/logger.rb +44 -0
  77. data/lib/legion/extensions/helpers/task.rb +60 -0
  78. data/lib/legion/extensions/helpers/transport.rb +44 -0
  79. data/lib/legion/extensions/transport.rb +159 -0
  80. data/lib/legion/lex.rb +89 -0
  81. data/lib/legion/process.rb +124 -0
  82. data/lib/legion/runner.rb +55 -0
  83. data/lib/legion/runner/log.rb +10 -0
  84. data/lib/legion/runner/status.rb +69 -0
  85. data/lib/legion/service.rb +130 -0
  86. data/lib/legion/supervision.rb +15 -0
  87. data/lib/legion/version.rb +3 -0
  88. metadata +239 -38
@@ -0,0 +1,32 @@
1
+ require_relative 'base'
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Actors
6
+ class Loop
7
+ include Concurrent::Async
8
+ include Legion::Extensions::Actors::Base
9
+
10
+ def initialize
11
+ @loop = true
12
+ async.run
13
+ rescue StandardError => e
14
+ Legion::Logging.error e
15
+ Legion::Logging.error e.backtrace
16
+ end
17
+
18
+ def run
19
+ action while @loop
20
+ end
21
+
22
+ def action(**_opts)
23
+ Legion::Logging.warn 'An extension is using the default action for a loop'
24
+ end
25
+
26
+ def cancel
27
+ @loop = false
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,15 @@
1
+ require_relative 'base'
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Actors
6
+ class Nothing
7
+ include Legion::Extensions::Actors::Base
8
+
9
+ def initialize; end
10
+
11
+ def cancel; end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,40 @@
1
+ require_relative 'base'
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Actors
6
+ class Once
7
+ include Legion::Extensions::Actors::Base
8
+
9
+ def initialize
10
+ return if disabled?
11
+
12
+ if respond_to? :functions
13
+ functions.each do
14
+ function
15
+ @task = Concurrent::ScheduledTask.execute(delay) do
16
+ use_runner ? runner : manual
17
+ end
18
+ end
19
+ else
20
+ @task = Concurrent::ScheduledTask.execute(delay) do
21
+ use_runner ? runner : manual
22
+ end
23
+ end
24
+ rescue StandardError => e
25
+ Legion::Logging.error e
26
+ end
27
+
28
+ def delay
29
+ 1.0
30
+ end
31
+
32
+ def cancel
33
+ return if disabled?
34
+
35
+ @task.cancel unless @task.cancelled?
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,87 @@
1
+ require_relative 'base'
2
+ require 'time'
3
+
4
+ module Legion
5
+ module Extensions
6
+ module Actors
7
+ class Poll
8
+ include Legion::Extensions::Actors::Base
9
+
10
+ def initialize # rubocop:disable Metrics/AbcSize
11
+ log.debug "Starting timer for #{self.class} with #{{ execution_interval: time, timeout_interval: timeout, run_now: run_now?, check_subtask: check_subtask? }}"
12
+ @timer = Concurrent::TimerTask.new(execution_interval: time, timeout_interval: timeout, run_now: run_now?) do
13
+ t1 = Time.now
14
+ log.debug "Running #{self.class}"
15
+ old_result = Legion::Cache.get(cache_name)
16
+ log.debug "Cached value for #{self.class}: #{old_result}"
17
+ results = Legion::JSON.load(Legion::JSON.dump(manual))
18
+ Legion::Cache.set(cache_name, results, time * 2)
19
+
20
+ unless old_result.nil?
21
+ results[:diff] = Hashdiff.diff(results, old_result, numeric_tolerance: 0.0, array_path: false) do |_path, obj1, obj2|
22
+ if int_percentage_normalize.positive? && obj1.is_a?(Integer) && obj2.is_a?(Integer)
23
+ obj1.between?(obj2 * (1 - int_percentage_normalize), obj2 * (1 + int_percentage_normalize))
24
+ end
25
+ end
26
+ results[:changed] = results[:diff].count.positive?
27
+
28
+ Legion::Logging.info results[:diff] if results[:changed]
29
+ Legion::Transport::Messages::CheckSubtask.new(runner_class: runner_class.to_s,
30
+ function: runner_function,
31
+ result: results,
32
+ type: 'poll_result',
33
+ polling: true).publish
34
+ end
35
+
36
+ sleep_time = 1 - (Time.now - t1)
37
+ sleep(sleep_time) if sleep_time.positive?
38
+ log.debug("#{self.class} result: #{results}")
39
+ results
40
+ rescue StandardError => e
41
+ Legion::Logging.fatal e.message
42
+ Legion::Logging.fatal e.backtrace
43
+ end
44
+ @timer.execute
45
+ rescue StandardError => e
46
+ Legion::Logging.error e.message
47
+ Legion::Logging.error e.backtrace
48
+ end
49
+
50
+ def cache_name
51
+ "#{lex_name}_#{runner_name}"
52
+ end
53
+
54
+ def int_percentage_normalize
55
+ 0.00
56
+ end
57
+
58
+ def time
59
+ 9
60
+ end
61
+
62
+ def run_now?
63
+ true
64
+ end
65
+
66
+ def check_subtask?
67
+ true
68
+ end
69
+
70
+ def timeout
71
+ 5
72
+ end
73
+
74
+ def action(_payload = {})
75
+ Legion::Logging.warn 'An extension is using the default block from Legion::Extensions::Runners::Every'
76
+ end
77
+
78
+ def cancel
79
+ Legion::Logging.debug 'Cancelling Legion Poller'
80
+ @timer.shutdown
81
+ rescue StandardError => e
82
+ Legion::Logging.error e.message
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,139 @@
1
+ require_relative 'base'
2
+ require 'date'
3
+
4
+ module Legion
5
+ module Extensions
6
+ module Actors
7
+ class Subscription
8
+ include Concurrent::Async
9
+ include Legion::Extensions::Actors::Base
10
+ include Legion::Extensions::Helpers::Transport
11
+
12
+ def initialize(**_options)
13
+ super()
14
+ @queue = queue.new
15
+ @queue.channel.prefetch(prefetch) if defined? prefetch
16
+ rescue StandardError => e
17
+ log.fatal e.message
18
+ log.fatal e.backtrace
19
+ end
20
+
21
+ def create_queue
22
+ queues.const_set(actor_const, Class.new(Legion::Transport::Queue))
23
+ exchange_object = default_exchange.new
24
+ queue_object = Kernel.const_get(queue_string).new
25
+
26
+ queue_object.bind(exchange_object, routing_key: actor_name)
27
+ queue_object.bind(exchange_object, routing_key: "#{lex_name}.#{actor_name}.#")
28
+ end
29
+
30
+ def queue
31
+ create_queue unless queues.const_defined?(actor_const)
32
+ Kernel.const_get queue_string
33
+ end
34
+
35
+ def queue_string
36
+ @queue_string ||= "#{queues}::#{actor_const}"
37
+ end
38
+
39
+ def cancel
40
+ return true unless @queue.channel.active
41
+
42
+ log.debug "Closing subscription to #{@queue.name}"
43
+ @consumer.cancel
44
+ @queue.channel.close
45
+ true
46
+ end
47
+
48
+ def block
49
+ false
50
+ end
51
+
52
+ def consumers
53
+ 1
54
+ end
55
+
56
+ def manual_ack
57
+ true
58
+ end
59
+
60
+ def delay_start
61
+ 0
62
+ end
63
+
64
+ def include_metadata_in_message?
65
+ true
66
+ end
67
+
68
+ def process_message(message, metadata, delivery_info)
69
+ payload = if metadata.content_encoding && metadata.content_encoding == 'encrypted/cs'
70
+ Legion::Crypt.decrypt(message, metadata.headers['iv'])
71
+ elsif metadata.content_encoding && metadata.content_encoding == 'encrypted/pk'
72
+ Legion::Crypt.decrypt_from_keypair(metadata.headers[:public_key], message)
73
+ else
74
+ message
75
+ end
76
+
77
+ message = if metadata.content_type == 'application/json'
78
+ Legion::JSON.load(payload)
79
+ else
80
+ { value: payload }
81
+ end
82
+ if include_metadata_in_message?
83
+ message = message.merge(metadata.headers.transform_keys(&:to_sym)) unless metadata.headers.nil?
84
+ message[:routing_key] = if Legion::Transport::TYPE == 'march_hare'
85
+ metadata.routing_key
86
+ else
87
+ delivery_info[:routing_key]
88
+ end
89
+ end
90
+
91
+ message[:timestamp] = (message[:timestamp_in_ms] / 1000).round if message.key?(:timestamp_in_ms) && !message.key?(:timestamp)
92
+ message[:datetime] = Time.at(message[:timestamp].to_i).to_datetime.to_s if message.key?(:timestamp)
93
+ message
94
+ end
95
+
96
+ def find_function(message = {})
97
+ return runner_function if actor_class.instance_methods(false).include?(:runner_function)
98
+ return function if actor_class.instance_methods(false).include?(:function)
99
+ return action if actor_class.instance_methods(false).include?(:action)
100
+ return message[:function] if message.key? :function
101
+
102
+ function
103
+ end
104
+
105
+ def subscribe
106
+ sleep(delay_start)
107
+ consumer_tag = "#{Legion::Settings[:client][:name]}_#{lex_name}_#{runner_name}_#{Thread.current.object_id}"
108
+ on_cancellation = block { cancel }
109
+
110
+ @consumer = @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag, on_cancellation: on_cancellation) do |*rmq_message|
111
+ payload = rmq_message.pop
112
+ metadata = rmq_message.last
113
+ delivery_info = rmq_message.first
114
+
115
+ message = process_message(payload, metadata, delivery_info)
116
+ if use_runner?
117
+ Legion::Runner.run(**message,
118
+ runner_class: runner_class,
119
+ function: find_function(message),
120
+ check_subtask: check_subtask?,
121
+ generate_task: generate_task?)
122
+ else
123
+ runner_class.send(find_function(message), **message)
124
+ end
125
+ @queue.acknowledge(delivery_info.delivery_tag) if manual_ack
126
+
127
+ cancel if Legion::Settings[:client][:shutting_down]
128
+ rescue StandardError => e
129
+ Legion::Logging.error e.message
130
+ Legion::Logging.error e.backtrace
131
+ Legion::Logging.error message
132
+ Legion::Logging.error function
133
+ @queue.reject(delivery_info.delivery_tag) if manual_ack
134
+ end
135
+ end
136
+ end
137
+ end
138
+ end
139
+ end
@@ -0,0 +1,61 @@
1
+ require_relative 'base'
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Builder
6
+ module Actors
7
+ include Legion::Extensions::Builder::Base
8
+
9
+ attr_reader :actors
10
+
11
+ def build_actors
12
+ @actors = {}
13
+ require_files(actor_files)
14
+ build_actor_list
15
+ build_meta_actor_list
16
+ end
17
+
18
+ def build_actor_list
19
+ actor_files.each do |file|
20
+ actor_name = file.split('/').last.sub('.rb', '')
21
+ actor_class = "#{lex_class}::Actor::#{actor_name.split('_').collect(&:capitalize).join}"
22
+ @actors[actor_name.to_sym] = {
23
+ extension: lex_class.to_s.downcase,
24
+ extension_name: extension_name,
25
+ actor_name: actor_name,
26
+ actor_class: Kernel.const_get(actor_class),
27
+ type: 'literal'
28
+ }
29
+ end
30
+ end
31
+
32
+ def build_meta_actor_list
33
+ @runners.each do |runner, attr|
34
+ next if @actors[runner.to_sym].is_a? Hash
35
+
36
+ actor_class = "#{attr[:extension_class]}::Actor::#{runner.to_s.split('_').collect(&:capitalize).join}"
37
+ build_meta_actor(runner, attr) unless Kernel.const_defined? actor_class
38
+ @actors[runner.to_sym] = {
39
+ extension: attr[:extension],
40
+ extension_name: attr[:extension_name],
41
+ actor_name: attr[:runner_name],
42
+ actor_class: Kernel.const_get(actor_class),
43
+ type: 'meta'
44
+ }
45
+ end
46
+ end
47
+
48
+ def build_meta_actor(runner, attr)
49
+ define_constant_two('Actor', root: lex_class)
50
+
51
+ Kernel.const_get("#{attr[:extension_class]}::Actor")
52
+ .const_set(runner.to_s.split('_').collect(&:capitalize).join, Class.new(Legion::Extensions::Actors::Subscription))
53
+ end
54
+
55
+ def actor_files
56
+ @actor_files ||= find_files('actors')
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,36 @@
1
+ module Legion
2
+ module Extensions
3
+ module Builder
4
+ module Base
5
+ def find_files(name, path = extension_path)
6
+ files = []
7
+ return files unless Dir.exist? "#{path}/#{name}"
8
+
9
+ Dir["#{path}/#{name}/*.rb"].each do |file|
10
+ files.push(file)
11
+ end
12
+ files
13
+ end
14
+
15
+ def require_files(files)
16
+ files.each { |file| require file }
17
+ end
18
+
19
+ def const_defined_two?(item, root = Kernel)
20
+ root.const_defined?(item.to_s)
21
+ end
22
+
23
+ def define_constant_two(item, root: Kernel, type: Module)
24
+ return true if const_defined?(item, root: root)
25
+
26
+ root.const_set(item.to_s, type.new)
27
+ end
28
+
29
+ def define_get(item, root: Kernel, type: Module)
30
+ define_constant_two(item, root: root, type: type) if const_defined_two?(item, root: root)
31
+ root.const_get(item)
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,24 @@
1
+ require_relative 'base'
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Builder
6
+ module Helpers
7
+ include Legion::Extensions::Builder::Base
8
+
9
+ def build_helpers
10
+ @helpers ||= []
11
+ @helpers.push(require_files(helper_files))
12
+ end
13
+
14
+ def helper_files
15
+ @helper_files ||= find_files('helpers')
16
+ end
17
+
18
+ def helpers
19
+ @helpers
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,58 @@
1
+ require_relative 'base'
2
+
3
+ module Legion
4
+ module Extensions
5
+ module Builder
6
+ module Runners
7
+ include Legion::Extensions::Builder::Base
8
+
9
+ attr_reader :runners
10
+
11
+ def build_runners
12
+ @runners = {}
13
+ lex_class.const_set('Runners', Module.new) unless lex_class.const_defined?('Runners')
14
+ require_files(runner_files)
15
+ build_runner_list
16
+ end
17
+
18
+ def build_runner_list
19
+ runner_files.each do |file|
20
+ runner_name = file.split('/').last.sub('.rb', '')
21
+ runner_class = "#{lex_class}::Runners::#{runner_name.split('_').collect(&:capitalize).join}"
22
+ loaded_runner = Kernel.const_get(runner_class)
23
+
24
+ @runners[runner_name.to_sym] = {
25
+ extension: lex_class.to_s.downcase,
26
+ extension_name: extension_name,
27
+ extension_class: lex_class,
28
+ runner_name: runner_name,
29
+ runner_class: runner_class,
30
+ runner_path: file,
31
+ class_methods: {}
32
+ }
33
+
34
+ if settings.key?(:runners) && settings[:runners].key?(runner_name.to_sym)
35
+ @runners[runner_name.to_sym][:desc] = settings[:runners][runner_name.to_sym][:desc]
36
+ end
37
+
38
+ loaded_runner.public_instance_methods(false).each do |runner_method|
39
+ @runners[runner_name.to_sym][:class_methods][runner_method] = {
40
+ args: loaded_runner.instance_method(runner_method).parameters
41
+ }
42
+ end
43
+
44
+ loaded_runner.methods(false).each do |runner_method|
45
+ @runners[runner_name.to_sym][:class_methods][runner_method] = {
46
+ args: loaded_runner.method(runner_method).parameters
47
+ }
48
+ end
49
+ end
50
+ end
51
+
52
+ def runner_files
53
+ @runner_files ||= find_files('runners')
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end