lepus 0.0.1.beta2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linter.yml +21 -0
- data/.github/workflows/specs.yml +93 -13
- data/.gitignore +2 -0
- data/.rubocop.yml +10 -0
- data/.tool-versions +1 -1
- data/Gemfile +7 -0
- data/Gemfile.lock +36 -9
- data/Makefile +19 -0
- data/README.md +562 -7
- data/bin/setup +5 -2
- data/config.ru +14 -0
- data/docker-compose.yml +5 -3
- data/docs/README.md +80 -0
- data/docs/cli.md +108 -0
- data/docs/configuration.md +171 -0
- data/docs/consumers.md +168 -0
- data/docs/getting-started.md +136 -0
- data/docs/images/lepus-web.png +0 -0
- data/docs/middleware.md +240 -0
- data/docs/producers.md +173 -0
- data/docs/prometheus.md +112 -0
- data/docs/rails.md +161 -0
- data/docs/supervisor.md +112 -0
- data/docs/testing.md +141 -0
- data/docs/web.md +85 -0
- data/examples/grafana-dashboard.json +450 -0
- data/gemfiles/Gemfile.rails-5.2 +7 -0
- data/gemfiles/{rails52.gemfile.lock → Gemfile.rails-5.2.lock} +102 -69
- data/gemfiles/Gemfile.rails-6.1 +7 -0
- data/gemfiles/{rails61.gemfile.lock → Gemfile.rails-6.1.lock} +113 -79
- data/gemfiles/{rails52.gemfile → Gemfile.rails-7.2} +1 -1
- data/gemfiles/Gemfile.rails-7.2.lock +321 -0
- data/gemfiles/{rails61.gemfile → Gemfile.rails-8.0} +1 -1
- data/gemfiles/Gemfile.rails-8.0.lock +322 -0
- data/lepus.gemspec +7 -1
- data/lib/lepus/cli.rb +35 -4
- data/lib/lepus/configuration.rb +107 -0
- data/lib/lepus/connection_pool.rb +135 -0
- data/lib/lepus/consumer.rb +59 -41
- data/lib/lepus/consumers/config.rb +183 -0
- data/lib/lepus/consumers/handler.rb +56 -0
- data/lib/lepus/consumers/middleware_chain.rb +22 -0
- data/lib/lepus/consumers/middlewares/exception_logger.rb +27 -0
- data/lib/lepus/consumers/middlewares/honeybadger.rb +33 -0
- data/lib/lepus/consumers/middlewares/json.rb +37 -0
- data/lib/lepus/consumers/middlewares/max_retry.rb +83 -0
- data/lib/lepus/consumers/middlewares/unique.rb +65 -0
- data/lib/lepus/consumers/stats.rb +70 -0
- data/lib/lepus/consumers/stats_registry.rb +29 -0
- data/lib/lepus/consumers/worker.rb +141 -0
- data/lib/lepus/consumers/worker_factory.rb +124 -0
- data/lib/lepus/consumers.rb +6 -0
- data/lib/lepus/message/delivery_info.rb +72 -0
- data/lib/lepus/message/metadata.rb +99 -0
- data/lib/lepus/message.rb +88 -5
- data/lib/lepus/middleware_chain.rb +83 -0
- data/lib/lepus/primitive/hash.rb +29 -0
- data/lib/lepus/process.rb +24 -24
- data/lib/lepus/process_registry/backend.rb +49 -0
- data/lib/lepus/process_registry/file_backend.rb +108 -0
- data/lib/lepus/process_registry/message_builder.rb +72 -0
- data/lib/lepus/process_registry/rabbitmq_backend.rb +153 -0
- data/lib/lepus/process_registry.rb +56 -23
- data/lib/lepus/processes/base.rb +0 -5
- data/lib/lepus/processes/callbacks.rb +3 -0
- data/lib/lepus/processes/interruptible.rb +4 -8
- data/lib/lepus/processes/procline.rb +1 -1
- data/lib/lepus/processes/registrable.rb +1 -1
- data/lib/lepus/processes/runnable.rb +1 -1
- data/lib/lepus/processes.rb +15 -0
- data/lib/lepus/producer.rb +141 -30
- data/lib/lepus/producers/config.rb +46 -0
- data/lib/lepus/producers/definition.rb +48 -0
- data/lib/lepus/producers/hooks.rb +170 -0
- data/lib/lepus/producers/middleware_chain.rb +22 -0
- data/lib/lepus/producers/middlewares/correlation_id.rb +37 -0
- data/lib/lepus/producers/middlewares/header.rb +47 -0
- data/lib/lepus/producers/middlewares/instrumentation.rb +30 -0
- data/lib/lepus/producers/middlewares/json.rb +47 -0
- data/lib/lepus/producers/middlewares/unique.rb +67 -0
- data/lib/lepus/producers.rb +7 -0
- data/lib/lepus/prometheus/collector.rb +149 -0
- data/lib/lepus/prometheus/instrumentation.rb +168 -0
- data/lib/lepus/prometheus.rb +48 -0
- data/lib/lepus/publisher.rb +67 -0
- data/lib/lepus/supervisor/children_pipes.rb +25 -0
- data/lib/lepus/supervisor/lifecycle_hooks.rb +50 -0
- data/lib/lepus/supervisor/pidfiled.rb +1 -1
- data/lib/lepus/supervisor/registry_cleaner.rb +22 -0
- data/lib/lepus/supervisor.rb +129 -25
- data/lib/lepus/testing/exchange.rb +95 -0
- data/lib/lepus/testing/message_builder.rb +177 -0
- data/lib/lepus/testing/rspec_matchers.rb +258 -0
- data/lib/lepus/testing.rb +210 -0
- data/lib/lepus/unique.rb +18 -0
- data/lib/lepus/version.rb +1 -1
- data/lib/lepus/web/aggregator.rb +154 -0
- data/lib/lepus/web/api.rb +132 -0
- data/lib/lepus/web/app.rb +37 -0
- data/lib/lepus/web/management_api.rb +192 -0
- data/lib/lepus/web/respond_with.rb +28 -0
- data/lib/lepus/web.rb +238 -0
- data/lib/lepus.rb +39 -28
- data/test_offline.html +189 -0
- data/web/assets/css/styles.css +635 -0
- data/web/assets/js/app.js +6 -0
- data/web/assets/js/bootstrap.js +20 -0
- data/web/assets/js/controllers/connection_controller.js +44 -0
- data/web/assets/js/controllers/dashboard_controller.js +499 -0
- data/web/assets/js/controllers/queue_controller.js +17 -0
- data/web/assets/js/controllers/theme_controller.js +31 -0
- data/web/assets/js/offline-manager.js +233 -0
- data/web/assets/js/service-worker-manager.js +65 -0
- data/web/index.html +159 -0
- data/web/sw.js +144 -0
- metadata +177 -18
- data/lib/lepus/consumer_config.rb +0 -149
- data/lib/lepus/consumer_wrapper.rb +0 -46
- data/lib/lepus/lifecycle_hooks.rb +0 -49
- data/lib/lepus/middlewares/honeybadger.rb +0 -23
- data/lib/lepus/middlewares/json.rb +0 -35
- data/lib/lepus/middlewares/max_retry.rb +0 -57
- data/lib/lepus/processes/consumer.rb +0 -113
- data/lib/lepus/supervisor/config.rb +0 -45
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Lepus::Processes
|
|
4
|
-
class Consumer < Base
|
|
5
|
-
include Runnable
|
|
6
|
-
|
|
7
|
-
attr_reader :consumer_class
|
|
8
|
-
|
|
9
|
-
def initialize(class_name:, **options)
|
|
10
|
-
@consumer_class = class_name
|
|
11
|
-
@consumer_class = Lepus::Primitive::String.new(@consumer_class).constantize if @consumer_class.is_a?(String)
|
|
12
|
-
|
|
13
|
-
super(**options)
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
def metadata
|
|
17
|
-
super.merge(consumer_class: consumer_class.to_s)
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def before_fork
|
|
21
|
-
return unless @consumer_class.respond_to?(:before_fork, true)
|
|
22
|
-
|
|
23
|
-
@consumer_class.send(:before_fork)
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def after_fork
|
|
27
|
-
return unless @consumer_class.respond_to?(:after_fork, true)
|
|
28
|
-
|
|
29
|
-
@consumer_class.send(:after_fork)
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
private
|
|
33
|
-
|
|
34
|
-
SLEEP_INTERVAL = 5
|
|
35
|
-
|
|
36
|
-
def run
|
|
37
|
-
wrap_in_app_executor do
|
|
38
|
-
setup_consumer! # initialize bunny consumer within the #run method to ensure the process is running in the correct thread
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
loop do
|
|
42
|
-
break if shutting_down?
|
|
43
|
-
|
|
44
|
-
wrap_in_app_executor do
|
|
45
|
-
interruptible_sleep(SLEEP_INTERVAL)
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
ensure
|
|
49
|
-
Lepus.instrument(:shutdown_process, process: self) do
|
|
50
|
-
run_process_callbacks(:shutdown) { shutdown }
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def shutdown
|
|
55
|
-
@subscriptions.to_a.each(&:cancel)
|
|
56
|
-
@channel&.close
|
|
57
|
-
@bunny&.close
|
|
58
|
-
|
|
59
|
-
super
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def set_procline
|
|
63
|
-
procline consumer_class.name
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def setup_consumer!
|
|
67
|
-
if consumer_class.config.nil?
|
|
68
|
-
raise Lepus::InvalidConsumerConfigError, "Consumer #{consumer_class.name} has no configuration"
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
@bunny = Thread.current[:lepus_bunny] || Lepus.config.create_connection
|
|
72
|
-
@channel = Thread.current[:lepus_channel] || begin
|
|
73
|
-
@bunny.create_channel(nil, 1, true).tap do |channel|
|
|
74
|
-
channel.prefetch(1) # @TODO make this configurable
|
|
75
|
-
channel.on_uncaught_exception { |error|
|
|
76
|
-
handle_thread_error(error)
|
|
77
|
-
}
|
|
78
|
-
end
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
@exchange = @channel.exchange(*consumer_class.config.exchange_args)
|
|
82
|
-
if (args = consumer_class.config.retry_queue_args)
|
|
83
|
-
@retry_queue = @channel.queue(*args)
|
|
84
|
-
end
|
|
85
|
-
if (args = consumer_class.config.error_queue_args)
|
|
86
|
-
@error_queue = @channel.queue(*args)
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
@subscriptions = Array.new((_threads = 1)) do |n| # may add multiple consumers in the future
|
|
90
|
-
main_queue = @channel.queue(*consumer_class.config.consumer_queue_args)
|
|
91
|
-
consumer_class.config.binds_args.each do |opts|
|
|
92
|
-
main_queue.bind(@exchange, **opts)
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
consumer_instance = consumer_class.new
|
|
96
|
-
consumer_wrapper = Lepus::ConsumerWrapper.new(
|
|
97
|
-
consumer_instance,
|
|
98
|
-
main_queue.channel,
|
|
99
|
-
main_queue,
|
|
100
|
-
"#{consumer_class.name}-#{n + 1}"
|
|
101
|
-
)
|
|
102
|
-
consumer_wrapper.on_delivery do |delivery_info, metadata, payload|
|
|
103
|
-
consumer_wrapper.process_delivery(delivery_info, metadata, payload)
|
|
104
|
-
end
|
|
105
|
-
main_queue.subscribe_with(consumer_wrapper)
|
|
106
|
-
end
|
|
107
|
-
rescue Bunny::TCPConnectionFailed, Bunny::PossibleAuthenticationFailureError
|
|
108
|
-
raise Lepus::ShutdownError
|
|
109
|
-
rescue Lepus::InvalidConsumerConfigError
|
|
110
|
-
raise Lepus::ShutdownError
|
|
111
|
-
end
|
|
112
|
-
end
|
|
113
|
-
end
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "yaml"
|
|
4
|
-
require "logger"
|
|
5
|
-
|
|
6
|
-
module Lepus
|
|
7
|
-
class Supervisor < Processes::Base
|
|
8
|
-
class Config
|
|
9
|
-
class ProcessStruct < Struct.new(:process_class, :attributes)
|
|
10
|
-
def instantiate
|
|
11
|
-
process_class.new(**attributes)
|
|
12
|
-
end
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
attr_accessor :pidfile, :require_file
|
|
16
|
-
|
|
17
|
-
def initialize(require_file: nil, pidfile: "tmp/pids/lepus.pid", **kwargs)
|
|
18
|
-
@pidfile = pidfile
|
|
19
|
-
@require_file = require_file
|
|
20
|
-
self.consumers = kwargs[:consumers] if kwargs.key?(:consumers)
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def configured_processes
|
|
24
|
-
consumer_processes
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def consumers=(vals)
|
|
28
|
-
@consumer_processes = nil
|
|
29
|
-
@consumers = Array(vals).map(&:to_s)
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def consumers
|
|
33
|
-
@consumers ||= Lepus::Consumer.descendants.reject(&:abstract_class?).map(&:name).compact
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
protected
|
|
37
|
-
|
|
38
|
-
def consumer_processes
|
|
39
|
-
@consumer_processes ||= consumers.map do |class_name|
|
|
40
|
-
ProcessStruct.new(Lepus::Processes::Consumer, {class_name: class_name})
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
end
|