karafka 1.4.0 → 2.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +89 -18
- data/.ruby-version +1 -1
- data/CHANGELOG.md +365 -1
- data/CONTRIBUTING.md +10 -19
- data/Gemfile +6 -0
- data/Gemfile.lock +56 -112
- data/LICENSE +17 -0
- data/LICENSE-COMM +89 -0
- data/LICENSE-LGPL +165 -0
- data/README.md +61 -68
- data/bin/benchmarks +85 -0
- data/bin/create_token +22 -0
- data/bin/integrations +272 -0
- data/bin/karafka +10 -0
- data/bin/scenario +29 -0
- data/bin/stress_many +13 -0
- data/bin/stress_one +13 -0
- data/certs/cert_chain.pem +26 -0
- data/certs/karafka-pro.pem +11 -0
- data/config/errors.yml +59 -38
- data/docker-compose.yml +10 -3
- data/karafka.gemspec +18 -21
- data/lib/active_job/karafka.rb +21 -0
- data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
- data/lib/karafka/active_job/consumer.rb +26 -0
- data/lib/karafka/active_job/dispatcher.rb +38 -0
- data/lib/karafka/active_job/job_extensions.rb +34 -0
- data/lib/karafka/active_job/job_options_contract.rb +21 -0
- data/lib/karafka/active_job/routing/extensions.rb +33 -0
- data/lib/karafka/admin.rb +63 -0
- data/lib/karafka/app.rb +15 -20
- data/lib/karafka/base_consumer.rb +197 -31
- data/lib/karafka/cli/info.rb +44 -10
- data/lib/karafka/cli/install.rb +22 -12
- data/lib/karafka/cli/server.rb +17 -42
- data/lib/karafka/cli.rb +4 -3
- data/lib/karafka/connection/client.rb +379 -89
- data/lib/karafka/connection/listener.rb +250 -38
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +84 -0
- data/lib/karafka/connection/pauses_manager.rb +46 -0
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/connection/rebalance_manager.rb +78 -0
- data/lib/karafka/contracts/base.rb +17 -0
- data/lib/karafka/contracts/config.rb +88 -11
- data/lib/karafka/contracts/consumer_group.rb +21 -184
- data/lib/karafka/contracts/consumer_group_topic.rb +35 -11
- data/lib/karafka/contracts/server_cli_options.rb +19 -18
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/env.rb +46 -0
- data/lib/karafka/errors.rb +21 -21
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/helpers/colorize.rb +20 -0
- data/lib/karafka/helpers/multi_delegator.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
- data/lib/karafka/instrumentation/logger.rb +6 -10
- data/lib/karafka/instrumentation/logger_listener.rb +174 -0
- data/lib/karafka/instrumentation/monitor.rb +13 -61
- data/lib/karafka/instrumentation/notifications.rb +53 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
- data/lib/karafka/instrumentation.rb +21 -0
- data/lib/karafka/licenser.rb +75 -0
- data/lib/karafka/messages/batch_metadata.rb +45 -0
- data/lib/karafka/messages/builders/batch_metadata.rb +39 -0
- data/lib/karafka/messages/builders/message.rb +39 -0
- data/lib/karafka/messages/builders/messages.rb +34 -0
- data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
- data/lib/karafka/messages/messages.rb +64 -0
- data/lib/karafka/{params → messages}/metadata.rb +4 -6
- data/lib/karafka/messages/seek.rb +9 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
- data/lib/karafka/pro/active_job/consumer.rb +46 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
- data/lib/karafka/pro/base_consumer.rb +107 -0
- data/lib/karafka/pro/contracts/base.rb +21 -0
- data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +69 -0
- data/lib/karafka/pro/loader.rb +76 -0
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/coordinator.rb +85 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +38 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/partitioner.rb +58 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
- data/lib/karafka/pro/routing/topic_extensions.rb +74 -0
- data/lib/karafka/pro.rb +13 -0
- data/lib/karafka/process.rb +1 -0
- data/lib/karafka/processing/coordinator.rb +103 -0
- data/lib/karafka/processing/coordinators_buffer.rb +54 -0
- data/lib/karafka/processing/executor.rb +126 -0
- data/lib/karafka/processing/executors_buffer.rb +88 -0
- data/lib/karafka/processing/jobs/base.rb +55 -0
- data/lib/karafka/processing/jobs/consume.rb +47 -0
- data/lib/karafka/processing/jobs/revoked.rb +22 -0
- data/lib/karafka/processing/jobs/shutdown.rb +23 -0
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/jobs_queue.rb +144 -0
- data/lib/karafka/processing/partitioner.rb +22 -0
- data/lib/karafka/processing/result.rb +37 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +91 -0
- data/lib/karafka/processing/workers_batch.rb +27 -0
- data/lib/karafka/railtie.rb +127 -0
- data/lib/karafka/routing/builder.rb +26 -23
- data/lib/karafka/routing/consumer_group.rb +37 -17
- data/lib/karafka/routing/consumer_mapper.rb +1 -2
- data/lib/karafka/routing/proxy.rb +9 -16
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +53 -0
- data/lib/karafka/routing/subscription_groups_builder.rb +54 -0
- data/lib/karafka/routing/topic.rb +65 -24
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +51 -0
- data/lib/karafka/serialization/json/deserializer.rb +6 -15
- data/lib/karafka/server.rb +67 -26
- data/lib/karafka/setup/config.rb +153 -175
- data/lib/karafka/status.rb +14 -5
- data/lib/karafka/templates/example_consumer.rb.erb +16 -0
- data/lib/karafka/templates/karafka.rb.erb +17 -55
- data/lib/karafka/time_trackers/base.rb +19 -0
- data/lib/karafka/time_trackers/pause.rb +92 -0
- data/lib/karafka/time_trackers/poll.rb +65 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +46 -16
- data.tar.gz.sig +0 -0
- metadata +145 -171
- metadata.gz.sig +0 -0
- data/.github/FUNDING.yml +0 -3
- data/MIT-LICENCE +0 -18
- data/certs/mensfeld.pem +0 -25
- data/lib/karafka/attributes_map.rb +0 -62
- data/lib/karafka/backends/inline.rb +0 -16
- data/lib/karafka/base_responder.rb +0 -226
- data/lib/karafka/cli/flow.rb +0 -48
- data/lib/karafka/code_reloader.rb +0 -67
- data/lib/karafka/connection/api_adapter.rb +0 -161
- data/lib/karafka/connection/batch_delegator.rb +0 -55
- data/lib/karafka/connection/builder.rb +0 -18
- data/lib/karafka/connection/message_delegator.rb +0 -36
- data/lib/karafka/consumers/batch_metadata.rb +0 -10
- data/lib/karafka/consumers/callbacks.rb +0 -71
- data/lib/karafka/consumers/includer.rb +0 -64
- data/lib/karafka/consumers/responders.rb +0 -24
- data/lib/karafka/consumers/single_params.rb +0 -15
- data/lib/karafka/contracts/responder_usage.rb +0 -54
- data/lib/karafka/fetcher.rb +0 -42
- data/lib/karafka/helpers/class_matcher.rb +0 -88
- data/lib/karafka/helpers/config_retriever.rb +0 -46
- data/lib/karafka/helpers/inflector.rb +0 -26
- data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
- data/lib/karafka/params/batch_metadata.rb +0 -26
- data/lib/karafka/params/builders/batch_metadata.rb +0 -30
- data/lib/karafka/params/builders/params.rb +0 -38
- data/lib/karafka/params/builders/params_batch.rb +0 -25
- data/lib/karafka/params/params_batch.rb +0 -60
- data/lib/karafka/patches/ruby_kafka.rb +0 -47
- data/lib/karafka/persistence/client.rb +0 -29
- data/lib/karafka/persistence/consumers.rb +0 -45
- data/lib/karafka/persistence/topics.rb +0 -48
- data/lib/karafka/responders/builder.rb +0 -36
- data/lib/karafka/responders/topic.rb +0 -55
- data/lib/karafka/routing/topic_mapper.rb +0 -53
- data/lib/karafka/serialization/json/serializer.rb +0 -31
- data/lib/karafka/setup/configurators/water_drop.rb +0 -36
- data/lib/karafka/templates/application_responder.rb.erb +0 -11
data/README.md
CHANGED
@@ -1,99 +1,92 @@
|
|
1
1
|
![karafka logo](https://raw.githubusercontent.com/karafka/misc/master/logo/karafka_logotype_transparent2.png)
|
2
2
|
|
3
|
-
[![Build Status](https://
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
If you are looking for the documentation for Karafka `1.3.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.3).
|
3
|
+
[![Build Status](https://github.com/karafka/karafka/actions/workflows/ci.yml/badge.svg)](https://github.com/karafka/karafka/actions/workflows/ci.yml)
|
4
|
+
[![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
|
5
|
+
[![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
|
8
6
|
|
9
7
|
## About Karafka
|
10
8
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup, and deployment along with other useful details on how to run Karafka.
|
9
|
+
Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
|
10
|
+
|
11
|
+
- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
|
12
|
+
- Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
|
13
|
+
- [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
|
14
|
+
- Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
|
15
|
+
- Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
|
16
|
+
- Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.
|
17
|
+
|
18
|
+
```ruby
|
19
|
+
# Define what topics you want to consume with which consumers in karafka.rb
|
20
|
+
Karafka::App.routes.draw do
|
21
|
+
topic 'system_events' do
|
22
|
+
consumer EventsConsumer
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# And create your consumers, within which your messages will be processed
|
27
|
+
class EventsConsumer < ApplicationConsumer
|
28
|
+
# Example that utilizes ActiveRecord#insert_all and Karafka batch processing
|
29
|
+
def consume
|
30
|
+
# Store all of the incoming Kafka events locally in an efficient way
|
31
|
+
Event.insert_all messages.payloads
|
32
|
+
end
|
33
|
+
end
|
34
|
+
```
|
39
35
|
|
40
|
-
|
36
|
+
Karafka **uses** threads to handle many messages simultaneously in the same process. It does not require Rails but will integrate tightly with any Ruby on Rails applications to make event processing dead simple.
|
41
37
|
|
42
38
|
## Getting started
|
43
39
|
|
44
|
-
If you're
|
40
|
+
If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
|
45
41
|
|
46
42
|
- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
|
47
|
-
- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with
|
48
|
-
|
49
|
-
If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
|
43
|
+
- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Rails and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
|
50
44
|
|
51
|
-
|
52
|
-
git clone https://github.com/karafka/example-app ./example_app
|
53
|
-
```
|
45
|
+
If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to visit our [Getting started](https://karafka.io/docs/Getting-Started) guides and the [example apps repository](https://github.com/karafka/example-apps).
|
54
46
|
|
55
|
-
|
56
|
-
|
57
|
-
```bash
|
58
|
-
cd ./example_app
|
59
|
-
bundle install
|
60
|
-
```
|
47
|
+
We also maintain many [integration specs](https://github.com/karafka/karafka/tree/master/spec/integrations) illustrating various use-cases and features of the framework.
|
61
48
|
|
62
|
-
|
49
|
+
### TL;DR (1 minute from setup to publishing and consuming messages)
|
63
50
|
|
64
|
-
**
|
51
|
+
**Prerequisites**: Kafka running. You can start it by following instructions from [here](https://karafka.io/docs/Setting-up-Kafka).
|
65
52
|
|
66
|
-
|
53
|
+
1. Add and install Karafka:
|
67
54
|
|
68
|
-
|
55
|
+
```bash
|
56
|
+
bundle add karafka
|
69
57
|
|
70
|
-
|
58
|
+
bundle exec karafka install
|
59
|
+
```
|
71
60
|
|
72
|
-
|
61
|
+
2. Dispatch a message to the example topic using the Rails or Ruby console:
|
73
62
|
|
74
|
-
|
75
|
-
|
76
|
-
|
63
|
+
```ruby
|
64
|
+
Karafka.producer.produce_sync(topic: 'example', payload: { 'ping' => 'pong' }.to_json)
|
65
|
+
```
|
77
66
|
|
78
|
-
|
67
|
+
3. Run Karafka server and see the consumption magic happen:
|
79
68
|
|
80
|
-
|
69
|
+
```bash
|
70
|
+
bundle exec karafka server
|
81
71
|
|
82
|
-
|
72
|
+
[7616dc24-505a-417f-b87b-6bf8fc2d98c5] Polled 1 message in 1000ms
|
73
|
+
[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example started
|
74
|
+
{"ping"=>"pong"}
|
75
|
+
[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example finished in 0ms
|
76
|
+
```
|
83
77
|
|
84
|
-
|
78
|
+
## Want to Upgrade? LGPL is not for you? Want to help?
|
85
79
|
|
86
|
-
|
80
|
+
I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
|
87
81
|
|
88
|
-
|
82
|
+
**20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
|
89
83
|
|
90
|
-
|
84
|
+
Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io) for more details.
|
91
85
|
|
92
|
-
|
93
|
-
<a href="https://github.com/karafka/karafka/graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>
|
86
|
+
## Support
|
94
87
|
|
95
|
-
|
88
|
+
Karafka has [Wiki pages](https://karafka.io/docs) for almost everything and a pretty decent [FAQ](https://karafka.io/docs/FAQ). It covers the installation, setup, and deployment, along with other useful details on how to run Karafka.
|
96
89
|
|
97
|
-
|
90
|
+
If you have questions about using Karafka, feel free to join our [Slack](https://slack.karafka.io) channel.
|
98
91
|
|
99
|
-
|
92
|
+
Karafka has [priority support](https://karafka.io/docs/Pro-Support) for technical and architectural questions that is part of the Karafka Pro subscription.
|
data/bin/benchmarks
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Runner for running given benchmark cases
|
4
|
+
# Some of the cases require pre-populated data and we populate this in places that need it
|
5
|
+
# In other cases we generate this data in a background process, so the partitions data stream
|
6
|
+
# is consistent and we don't end up consuming huge batches of a single partition.
|
7
|
+
|
8
|
+
require 'open3'
|
9
|
+
require 'pathname'
|
10
|
+
|
11
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
|
13
|
+
|
14
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
15
|
+
|
16
|
+
# Load all the benchmarks
|
17
|
+
benchmarks = Dir[ROOT_PATH.join('spec/benchmarks/**/*.rb')]
|
18
|
+
|
19
|
+
# If filter is provided, apply
|
20
|
+
benchmarks.delete_if { |name| !name.include?(ARGV[0]) } if ARGV[0]
|
21
|
+
|
22
|
+
raise ArgumentError, "No benchmarks with filter: #{ARGV[0]}" if benchmarks.empty?
|
23
|
+
|
24
|
+
# We may skip seeding if we are running the benchmarks multiple times, then since we do not
|
25
|
+
# commit offsets we can skip generating more data
|
26
|
+
if ENV['SEED']
|
27
|
+
require 'spec/benchmarks_helper'
|
28
|
+
|
29
|
+
# We need to setup karafka here to have producer for data seeding
|
30
|
+
setup_karafka
|
31
|
+
|
32
|
+
# This takes some time but needs to run only once per benchmark session
|
33
|
+
puts 'Seeding benchmarks data...'
|
34
|
+
|
35
|
+
producer = Karafka::App.producer
|
36
|
+
|
37
|
+
# We make our data json compatible so we can also benchmark serialization
|
38
|
+
elements = Array.new(100_000) { { a: :b }.to_json }
|
39
|
+
|
40
|
+
# We do not populate data of benchmarks_0_10 as we use it with life-stream data only
|
41
|
+
%w[
|
42
|
+
benchmarks_00_01
|
43
|
+
benchmarks_00_05
|
44
|
+
].each do |topic_name|
|
45
|
+
partitions_count = topic_name.split('_').last.to_i
|
46
|
+
|
47
|
+
partitions_count.times do |partition|
|
48
|
+
puts "Seeding #{topic_name}:#{partition}"
|
49
|
+
|
50
|
+
elements.each_slice(10_000) do |data_slice|
|
51
|
+
data = data_slice.map do |data|
|
52
|
+
{ topic: topic_name, payload: data, partition: partition }
|
53
|
+
end
|
54
|
+
|
55
|
+
producer.buffer_many(data)
|
56
|
+
producer.flush_sync
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Selects requested benchmarks and runs them one after another
|
63
|
+
benchmarks.each do |benchmark_path|
|
64
|
+
puts "Running #{benchmark_path.gsub("#{ROOT_PATH}/spec/benchmarks/", '')}"
|
65
|
+
|
66
|
+
benchmark = "bundle exec ruby -r ./spec/benchmarks_helper.rb #{benchmark_path}"
|
67
|
+
|
68
|
+
Open3.popen3(benchmark) do |stdin, stdout, stderr, thread|
|
69
|
+
t1 = Thread.new do
|
70
|
+
while line = stdout.gets
|
71
|
+
puts(line)
|
72
|
+
end
|
73
|
+
rescue IOError
|
74
|
+
end
|
75
|
+
|
76
|
+
t2 = Thread.new do
|
77
|
+
while line = stderr.gets
|
78
|
+
puts(line)
|
79
|
+
end
|
80
|
+
rescue IOError
|
81
|
+
end
|
82
|
+
|
83
|
+
thread.join
|
84
|
+
end
|
85
|
+
end
|
data/bin/create_token
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'openssl'
|
4
|
+
require 'base64'
|
5
|
+
require 'json'
|
6
|
+
require 'date'
|
7
|
+
|
8
|
+
PRIVATE_KEY_LOCATION = File.join(Dir.home, '.ssh', 'karafka-pro', 'id_rsa')
|
9
|
+
|
10
|
+
# Name of the entity that acquires the license
|
11
|
+
ENTITY = ARGV[0]
|
12
|
+
|
13
|
+
raise ArgumentError, 'Entity missing' if ENTITY.nil? || ENTITY.empty?
|
14
|
+
|
15
|
+
pro_token_data = { entity: ENTITY }
|
16
|
+
|
17
|
+
# This code uses my private key to generate a new token for Karafka Pro capabilities
|
18
|
+
private_key = OpenSSL::PKey::RSA.new(File.read(PRIVATE_KEY_LOCATION))
|
19
|
+
|
20
|
+
bin_key = private_key.private_encrypt(pro_token_data.to_json)
|
21
|
+
|
22
|
+
puts Base64.encode64(bin_key)
|
data/bin/integrations
ADDED
@@ -0,0 +1,272 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Runner to run integration specs in parallel
|
4
|
+
|
5
|
+
# Part of integration specs run linear without bundler.
|
6
|
+
# If we would run bundle exec when running this code, bundler would inject its own context
|
7
|
+
# into them, messing things up heavily
|
8
|
+
#
|
9
|
+
# Types of specs:
|
10
|
+
# - regular - can run in parallel, includes all the helpers
|
11
|
+
# - pristine - cannot run in parallel, uses custom bundler but includes helpers
|
12
|
+
# - poro - cannot run in parallel, uses custom bundler, does not include any helpers
|
13
|
+
raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
|
14
|
+
|
15
|
+
require 'open3'
|
16
|
+
require 'fileutils'
|
17
|
+
require 'pathname'
|
18
|
+
require 'tmpdir'
|
19
|
+
require 'etc'
|
20
|
+
|
21
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
22
|
+
|
23
|
+
# How many child processes with integration specs do we want to run in parallel
|
24
|
+
# When the value is high, there's a problem with thread allocation on Github CI, tht is why
|
25
|
+
# we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
|
26
|
+
# of CPU
|
27
|
+
CONCURRENCY = ENV.key?('CI') ? 3 : Etc.nprocessors * 2
|
28
|
+
|
29
|
+
# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
|
30
|
+
MAX_BUFFER_OUTPUT = 51_200
|
31
|
+
|
32
|
+
# Abstraction around a single test scenario execution process
|
33
|
+
class Scenario
|
34
|
+
# How long a scenario can run before we kill it
|
35
|
+
# This is a fail-safe just in case something would hang
|
36
|
+
MAX_RUN_TIME = 5 * 60 # 5 minutes tops
|
37
|
+
|
38
|
+
# There are rare cases where Karafka may force shutdown for some of the integration cases
|
39
|
+
# This includes exactly those
|
40
|
+
EXIT_CODES = {
|
41
|
+
default: [0],
|
42
|
+
'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
|
43
|
+
'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
|
44
|
+
'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
|
45
|
+
'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
|
46
|
+
}.freeze
|
47
|
+
|
48
|
+
private_constant :MAX_RUN_TIME, :EXIT_CODES
|
49
|
+
|
50
|
+
# Creates scenario instance and runs in the background process
|
51
|
+
#
|
52
|
+
# @param path [String] path to the scenarios file
|
53
|
+
def initialize(path)
|
54
|
+
@path = path
|
55
|
+
# First 1024 characters from stdout
|
56
|
+
@stdout_head = ''
|
57
|
+
# Last 1024 characters from stdout
|
58
|
+
@stdout_tail = ''
|
59
|
+
end
|
60
|
+
|
61
|
+
# Starts running given scenario in a separate process
|
62
|
+
def start
|
63
|
+
@stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
|
64
|
+
@started_at = current_time
|
65
|
+
end
|
66
|
+
|
67
|
+
# @return [String] integration spec name
|
68
|
+
def name
|
69
|
+
@path.gsub("#{ROOT_PATH}/spec/integrations/", '')
|
70
|
+
end
|
71
|
+
|
72
|
+
# @return [Symbol] type of spec
|
73
|
+
def type
|
74
|
+
scenario_dir = File.dirname(@path)
|
75
|
+
|
76
|
+
return :poro if scenario_dir.end_with?('_poro')
|
77
|
+
return :pristine if scenario_dir.end_with?('_pristine')
|
78
|
+
|
79
|
+
:regular
|
80
|
+
end
|
81
|
+
|
82
|
+
# @return [Boolean] any spec that is not a regular one should not run in parallel with others
|
83
|
+
def linear?
|
84
|
+
type != :regular
|
85
|
+
end
|
86
|
+
|
87
|
+
# @return [Boolean] did this scenario finished or is it still running
|
88
|
+
def finished?
|
89
|
+
# If the thread is running too long, kill it
|
90
|
+
if current_time - @started_at > MAX_RUN_TIME
|
91
|
+
begin
|
92
|
+
Process.kill('TERM', pid)
|
93
|
+
# It may finish right after we want to kill it, that's why we ignore this
|
94
|
+
rescue Errno::ESRCH
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# We read it so it won't grow as we use our default logger that prints to both test.log and
|
99
|
+
# to stdout. Otherwise after reaching the buffer size, it would hang
|
100
|
+
buffer = ''
|
101
|
+
@stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
|
102
|
+
@stdout_head = buffer if @stdout_head.empty?
|
103
|
+
@stdout_tail << buffer
|
104
|
+
@stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
|
105
|
+
|
106
|
+
!@wait_thr.alive?
|
107
|
+
end
|
108
|
+
|
109
|
+
# @return [Boolean] did this scenario finish successfully or not
|
110
|
+
def success?
|
111
|
+
expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
|
112
|
+
|
113
|
+
expected_exit_codes.include?(exit_code)
|
114
|
+
end
|
115
|
+
|
116
|
+
# @return [Integer] pid of the process of this scenario
|
117
|
+
def pid
|
118
|
+
@wait_thr.pid
|
119
|
+
end
|
120
|
+
|
121
|
+
# @return [Integer] exit code of the process running given scenario
|
122
|
+
def exit_code
|
123
|
+
# There may be no exit status if we killed the thread
|
124
|
+
@wait_thr.value&.exitstatus || 123
|
125
|
+
end
|
126
|
+
|
127
|
+
# @return [String] exit status of the process
|
128
|
+
def exit_status
|
129
|
+
@wait_thr.value.to_s
|
130
|
+
end
|
131
|
+
|
132
|
+
# Prints a status report when scenario is finished and stdout if it failed
|
133
|
+
def report
|
134
|
+
if success?
|
135
|
+
print "\e[#{32}m#{'.'}\e[0m"
|
136
|
+
else
|
137
|
+
buffer = ''
|
138
|
+
|
139
|
+
@stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
|
140
|
+
|
141
|
+
puts
|
142
|
+
puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
|
143
|
+
puts "Time taken: #{current_time - @started_at} seconds"
|
144
|
+
puts "Exit code: #{exit_code}"
|
145
|
+
puts "Exit status: #{exit_status}"
|
146
|
+
puts @stdout_head
|
147
|
+
puts '...'
|
148
|
+
puts @stdout_tail
|
149
|
+
puts buffer
|
150
|
+
puts
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
|
156
|
+
# Sets up a proper environment for a given spec to run and returns the run command
|
157
|
+
# @return [String] run command
|
158
|
+
def init_and_build_cmd
|
159
|
+
case type
|
160
|
+
when :poro
|
161
|
+
scenario_dir = File.dirname(@path)
|
162
|
+
# We copy the spec into a temp dir, not to pollute the spec location with logs, etc
|
163
|
+
temp_dir = Dir.mktmpdir
|
164
|
+
file_name = File.basename(@path)
|
165
|
+
|
166
|
+
FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
|
167
|
+
|
168
|
+
<<~CMD
|
169
|
+
cd #{temp_dir} &&
|
170
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
171
|
+
BUNDLE_AUTO_INSTALL=true \
|
172
|
+
PRISTINE_MODE=true \
|
173
|
+
bundle exec ruby #{file_name}
|
174
|
+
CMD
|
175
|
+
when :pristine
|
176
|
+
scenario_dir = File.dirname(@path)
|
177
|
+
# We copy the spec into a temp dir, not to pollute the spec location with logs, etc
|
178
|
+
temp_dir = Dir.mktmpdir
|
179
|
+
file_name = File.basename(@path)
|
180
|
+
|
181
|
+
FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
|
182
|
+
|
183
|
+
<<~CMD
|
184
|
+
cd #{temp_dir} &&
|
185
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
186
|
+
BUNDLE_AUTO_INSTALL=true \
|
187
|
+
PRISTINE_MODE=true \
|
188
|
+
bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{file_name}
|
189
|
+
CMD
|
190
|
+
else
|
191
|
+
<<~CMD
|
192
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
193
|
+
bundle exec ruby -r ./spec/integrations_helper.rb #{@path}
|
194
|
+
CMD
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
# @return [Float] current machine time
|
199
|
+
def current_time
|
200
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
# Load all the specs
|
205
|
+
specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
|
206
|
+
|
207
|
+
# If filters is provided, apply
|
208
|
+
# Allows to provide several filters one after another and applies all of them
|
209
|
+
ARGV.each do |filter|
|
210
|
+
specs.delete_if { |name| !name.include?(filter) }
|
211
|
+
end
|
212
|
+
|
213
|
+
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
214
|
+
|
215
|
+
# Randomize order
|
216
|
+
seed = (ENV['SEED'] || rand(0..10_000)).to_i
|
217
|
+
|
218
|
+
puts "Random seed: #{seed}"
|
219
|
+
|
220
|
+
scenarios = specs
|
221
|
+
.shuffle(random: Random.new(seed))
|
222
|
+
.map { |integration_test| Scenario.new(integration_test) }
|
223
|
+
|
224
|
+
regulars = scenarios.reject(&:linear?)
|
225
|
+
linears = scenarios - regulars
|
226
|
+
|
227
|
+
active_scenarios = []
|
228
|
+
finished_scenarios = []
|
229
|
+
|
230
|
+
while finished_scenarios.size < scenarios.size
|
231
|
+
# If we have space to run another scenario, we add it
|
232
|
+
if active_scenarios.size < CONCURRENCY
|
233
|
+
scenario = nil
|
234
|
+
# We can run only one linear at the same time due to concurrency issues within bundler
|
235
|
+
# Since they usually take longer than others, we try to run them as fast as possible when there
|
236
|
+
# is a slot
|
237
|
+
scenario = linears.pop unless active_scenarios.any?(&:linear?)
|
238
|
+
scenario ||= regulars.pop
|
239
|
+
|
240
|
+
if scenario
|
241
|
+
scenario.start
|
242
|
+
active_scenarios << scenario
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
active_scenarios.select(&:finished?).each do |exited|
|
247
|
+
scenario = active_scenarios.delete(exited)
|
248
|
+
scenario.report
|
249
|
+
finished_scenarios << scenario
|
250
|
+
end
|
251
|
+
|
252
|
+
sleep(0.1)
|
253
|
+
end
|
254
|
+
|
255
|
+
failed_scenarios = finished_scenarios.reject(&:success?)
|
256
|
+
|
257
|
+
# Report once more on the failed jobs
|
258
|
+
# This will only list scenarios that failed without printing their stdout here.
|
259
|
+
if failed_scenarios.empty?
|
260
|
+
puts
|
261
|
+
else
|
262
|
+
puts "\nFailed scenarios:\n\n"
|
263
|
+
|
264
|
+
failed_scenarios.each do |scenario|
|
265
|
+
puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
|
266
|
+
end
|
267
|
+
|
268
|
+
puts
|
269
|
+
|
270
|
+
# Exit with 1 if not all scenarios were successful
|
271
|
+
exit 1
|
272
|
+
end
|
data/bin/karafka
CHANGED
@@ -2,9 +2,19 @@
|
|
2
2
|
|
3
3
|
require 'karafka'
|
4
4
|
|
5
|
+
# We set this to indicate, that the process in which we are (whatever it does) was started using
|
6
|
+
# our bin/karafka cli
|
7
|
+
ENV['KARAFKA_CLI'] = 'true'
|
8
|
+
|
5
9
|
# If there is a boot file, we need to require it as we expect it to contain
|
6
10
|
# Karafka app setup, routes, etc
|
7
11
|
if File.exist?(Karafka.boot_file)
|
12
|
+
rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
|
13
|
+
|
14
|
+
# Load Rails environment file that starts Rails, so we can reference consumers and other things
|
15
|
+
# from `karafka.rb` file. This will work only for Rails, for non-rails a manual setup is needed
|
16
|
+
require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
|
17
|
+
|
8
18
|
require Karafka.boot_file.to_s
|
9
19
|
else
|
10
20
|
# However when it is unavailable, we still want to be able to run help command
|
data/bin/scenario
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Runner for non-parallel execution of a single scenario.
|
4
|
+
# It prints all the info stdout, etc and basically replaces itself with the scenario execution.
|
5
|
+
# It is useful when we work with a single spec and we need all the debug info
|
6
|
+
|
7
|
+
raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
|
8
|
+
|
9
|
+
require 'open3'
|
10
|
+
require 'fileutils'
|
11
|
+
require 'pathname'
|
12
|
+
require 'tmpdir'
|
13
|
+
require 'etc'
|
14
|
+
|
15
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
16
|
+
|
17
|
+
# Load all the specs
|
18
|
+
specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
|
19
|
+
|
20
|
+
# If filters is provided, apply
|
21
|
+
# Allows to provide several filters one after another and applies all of them
|
22
|
+
ARGV.each do |filter|
|
23
|
+
specs.delete_if { |name| !name.include?(filter) }
|
24
|
+
end
|
25
|
+
|
26
|
+
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
27
|
+
raise ArgumentError, "Many specs found with filters: #{ARGV.join(', ')}" if specs.size != 1
|
28
|
+
|
29
|
+
exec("bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{specs[0]}")
|
data/bin/stress_many
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Runs integration specs in an endless loop
|
4
|
+
# This allows us to ensure (after long enough time) that the integrations test suit is stable and
|
5
|
+
# that there are no anomalies when running it for a long period of time
|
6
|
+
|
7
|
+
set -e
|
8
|
+
|
9
|
+
while :
|
10
|
+
do
|
11
|
+
clear
|
12
|
+
bin/integrations $1
|
13
|
+
done
|
data/bin/stress_one
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Runs a single integration spec in an endless loop
|
4
|
+
# This allows us to ensure (after long enough time) that the integration spec is stable and
|
5
|
+
# that there are no anomalies when running it for a long period of time
|
6
|
+
|
7
|
+
set -e
|
8
|
+
|
9
|
+
while :
|
10
|
+
do
|
11
|
+
clear
|
12
|
+
bin/scenario $1
|
13
|
+
done
|
@@ -0,0 +1,26 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
3
|
+
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
4
|
+
MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
|
5
|
+
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
6
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
|
7
|
+
jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
|
8
|
+
F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
|
9
|
+
ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
|
10
|
+
lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
|
11
|
+
rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
|
12
|
+
KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
|
13
|
+
Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
|
14
|
+
cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
|
15
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
|
16
|
+
iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
17
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
|
18
|
+
G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
|
19
|
+
vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
|
20
|
+
yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
|
21
|
+
EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
|
22
|
+
Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
|
23
|
+
KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
|
24
|
+
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
25
|
+
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
26
|
+
-----END CERTIFICATE-----
|
@@ -0,0 +1,11 @@
|
|
1
|
+
-----BEGIN RSA PUBLIC KEY-----
|
2
|
+
MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
|
3
|
+
tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
|
4
|
+
ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
|
5
|
+
Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
|
6
|
+
8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
|
7
|
+
BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
|
8
|
+
2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
|
9
|
+
2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
|
10
|
+
aQtjlQWKPx5xAgMBAAE=
|
11
|
+
-----END RSA PUBLIC KEY-----
|