rimless 0.3.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4d97aaf1dd758a38f0537d4d2419052241412cba8f7ff45c30d40d5a5254a6e3
4
- data.tar.gz: 483c104c176c020d79bec4347789be3d6558f21777571a36ffe5956813a75dff
3
+ metadata.gz: 89eb85ed138758a0b67496d3816fe748d7119eb9afcd9e5d08c2e116d24b28f8
4
+ data.tar.gz: dea1976e01d8719aee285af77f40ec140622d15997996bbffa1f34ba61949c99
5
5
  SHA512:
6
- metadata.gz: deb7b05ed932b8dee372ecf9812cff38bd851e8d82cad80d31615278d182c0d4fdc258dc1e974f55d3330a2696ac803faf95156c54f7c5c8948eecd9ef15c50a
7
- data.tar.gz: 8c3191a6e4e15240a72d36511143b775dce8602c375727defca0c2e11d6c56b6df7471a3ba2d70f95a5dc23e2eefbddfe74414c651904e7fd1fcf5b9dc2d9baa
6
+ metadata.gz: a26b32866b7a5a52a5921ee07f7d400051773d28a45b62eb536b9febe704879587ddcf69a8dabf25e5799d3367c789e4767e9525c6be7dea59ef07c3cb7fdbfe
7
+ data.tar.gz: b13f1768f869ed85c7ea27ef83c1667999e5503a2c9c400406c2348493a869136c6c7ba528f02e2b815a8eb9bf4325359e1e2fdc41951ffaefe6ce92bd1f072c
data/.rubocop.yml CHANGED
@@ -8,12 +8,12 @@ Documentation:
8
8
 
9
9
  AllCops:
10
10
  DisplayCopNames: true
11
- TargetRubyVersion: 2.3
11
+ TargetRubyVersion: 2.5
12
12
  Exclude:
13
13
  - bin/**/*
14
14
  - vendor/**/*
15
15
  - build/**/*
16
- - gemfiles/vendor/**/*
16
+ - gemfiles/**/*
17
17
 
18
18
  Metrics/BlockLength:
19
19
  Exclude:
data/.travis.yml CHANGED
@@ -6,18 +6,26 @@ sudo: false
6
6
  language: ruby
7
7
  cache: bundler
8
8
  rvm:
9
+ - 2.7
9
10
  - 2.6
10
11
  - 2.5
11
- - 2.4
12
- - 2.3
13
12
 
14
13
  gemfile:
15
14
  - gemfiles/rails_4.2.gemfile
16
15
  - gemfiles/rails_5.0.gemfile
17
16
  - gemfiles/rails_5.1.gemfile
18
17
  - gemfiles/rails_5.2.gemfile
18
+ - gemfiles/rails_6.0.gemfile
19
19
 
20
20
  before_install: gem install bundler
21
+ install:
22
+ # Rails 4 is not Ruby 2.7 compatible, so we skip this build
23
+ - |
24
+ [[ "${BUNDLE_GEMFILE}" =~ rails_4 && \
25
+ "${TRAVIS_RUBY_VERSION}" =~ 2.7 ]] && exit || true
26
+ # Regular build
27
+ - bundle install --jobs=3 --retry=3
28
+
21
29
  before_script:
22
30
  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
23
31
  - chmod +x ./cc-test-reporter
data/Appraisals CHANGED
@@ -19,3 +19,8 @@ appraise 'rails-5.2' do
19
19
  gem 'activesupport', '~> 5.2.2'
20
20
  gem 'railties', '~> 5.2.2'
21
21
  end
22
+
23
+ appraise 'rails-6.0' do
24
+ gem 'activesupport', '~> 6.0.0'
25
+ gem 'railties', '~> 6.0.0'
26
+ end
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ### 1.0.0
2
+
3
+ * Dropped support for Ruby 2.3/2.4 and added support for Rails 6.0 (#6)
4
+ * Implemented a simple opinionated Kafka consumer setup (#7)
5
+
1
6
  ### 0.3.0
2
7
 
3
8
  * Upgraded the avro_turf gem (`~> 0.11.0`) (#5)
data/Dockerfile CHANGED
@@ -1,4 +1,4 @@
1
- FROM hausgold/ruby:2.3
1
+ FROM hausgold/ruby:2.5
2
2
  MAINTAINER Hermann Mayer <hermann.mayer@hausgold.de>
3
3
 
4
4
  # Update system gem
data/Makefile CHANGED
@@ -26,12 +26,13 @@ RM ?= rm
26
26
  XARGS ?= xargs
27
27
 
28
28
  # Container binaries
29
- BUNDLE ?= bundle
30
29
  APPRAISAL ?= appraisal
30
+ BUNDLE ?= bundle
31
+ GEM ?= gem
31
32
  RAKE ?= rake
32
- YARD ?= yard
33
33
  RAKE ?= rake
34
34
  RUBOCOP ?= rubocop
35
+ YARD ?= yard
35
36
 
36
37
  # Files
37
38
  GEMFILES ?= $(subst _,-,$(patsubst $(GEMFILES_DIR)/%.gemfile,%,\
@@ -75,6 +76,8 @@ install:
75
76
  # Install the dependencies
76
77
  @$(MKDIR) -p $(VENDOR_DIR)
77
78
  @$(call run-shell,$(BUNDLE) check || $(BUNDLE) install --path $(VENDOR_DIR))
79
+ @$(call run-shell,GEM_HOME=vendor/bundle/ruby/$${RUBY_MAJOR}.0 \
80
+ $(GEM) install bundler -v "~> 1.0")
78
81
  @$(call run-shell,$(BUNDLE) exec $(APPRAISAL) install)
79
82
 
80
83
  update: install
data/README.md CHANGED
@@ -23,6 +23,11 @@ opinionated framework which sets up solid conventions for producing messages.
23
23
  - [Confluent Schema Registry Subject](#confluent-schema-registry-subject)
24
24
  - [Organize and write schema definitions](#organize-and-write-schema-definitions)
25
25
  - [Producing messages](#producing-messages)
26
+ - [Consuming messages](#consuming-messages)
27
+ - [Routing messages to consumers](#routing-messages-to-consumers)
28
+ - [Consuming event messages](#consuming-event-messages)
29
+ - [Listing consumer routes](#listing-consumer-routes)
30
+ - [Starting the consumer process(es)](#starting-the-consumer-processes)
26
31
  - [Encoding/Decoding messages](#encodingdecoding-messages)
27
32
  - [Handling of schemaless deep blobs](#handling-of-schemaless-deep-blobs)
28
33
  - [Writing tests for your messages](#writing-tests-for-your-messages)
@@ -80,6 +85,9 @@ Rimless.configure do |conf|
80
85
  # The Confluent Schema Registry API URL,
81
86
  # set to HAUSGOLD defaults when not set
82
87
  conf.schema_registry_url = 'http://your.schema-registry.local'
88
+
89
+ # The Sidekiq job queue to use for consuming jobs
90
+ config.consumer_job_queue = 'default'
83
91
  end
84
92
  ```
85
93
 
@@ -98,6 +106,7 @@ available configuration options:
98
106
  * **KAFKA_CLIENT_ID**: The Apache Kafka client identifier, falls back the the local application name.
99
107
  * **KAFKA_BROKERS**: A comma separated list of Apache Kafka brokers for cluster discovery (Plaintext, no-auth/no-SSL only for now) (eg. `kafka://your.domain:9092,kafka..`)
100
108
  * **KAFKA_SCHEMA_REGISTRY_URL**: The Confluent Schema Registry API URL to use for schema registrations.
109
+ * **KAFKA_SIDEKIQ_JOB_QUEUE**: The Sidekiq job queue to use for consuming jobs. Falls back to `default`.
101
110
 
102
111
  ### Conventions
103
112
 
@@ -279,6 +288,138 @@ Rimless.message(data: user, schema: :user_v1,
279
288
  Rimless.async_raw_message(data: encoded, topic: :users)
280
289
  ```
281
290
 
291
+ ### Consuming messages
292
+
293
+ The rimless gem makes it super easy to build consumer logic right into your
294
+ (Rails, standalone) application by utilizing the [Karafka
295
+ framework](https://github.com/karafka/karafka) under the hood. When you have
296
+ the rimless gem already installed you are ready to rumble to setup your
297
+ application to consume Apache Kafka messages. Just run the `$ rake
298
+ rimless:install` and all the consuming setup is done for you.
299
+
300
+ Afterwards you find the `karafka.rb` file at the root of your project together
301
+ with an example consumer (including specs). The default configuration follows
302
+ the base conventions and ships some opinions on the architecture. The
303
+ architecture looks like this:
304
+
305
+ ```
306
+ +----[Apache Kafka]
307
+ |
308
+ fetch message batches
309
+ |
310
+ v
311
+ +-----------------------------+
312
+ | Karafka/Rimless Consumer | +--------------------------------------+
313
+ | Shares a single consumer |--->| Sidekiq |
314
+ | group, multiple processes | | Runs the consumer class (children |
315
+ +-----------------------------+ | of Rimless::BaseConsumer) for each |
316
+ | message (Rimless::ConsumerJob), |
317
+ | one message per job |
318
+ +--------------------------------------+
319
+ ```
320
+
321
+ This architecture allows the consumer process to run mostly non-blocking and
322
+ the messages can be processed concurrently via Sidekiq. (including the error
323
+ handling and retrying)
324
+
325
+ #### Routing messages to consumers
326
+
327
+ The `karafka.rb` file at the root of your project is dedicated to configure the
328
+ consumer process, including the routing table. The routing is as easy as it
329
+ gets by following this pattern: `topic => consumer`. Here comes a the full
330
+ examples:
331
+
332
+ ```ruby
333
+ # Setup the topic-consumer routing table and boot the consumer application
334
+ Rimless.consumer.topics(
335
+ { app: :your_app, name: :your_topic } => CustomConsumer
336
+ ).boot!
337
+ ```
338
+
339
+ The key side of the hash is anything which is understood by the `Rimless.topic`
340
+ method. With one addition: you can change `:name` to `:names` and pass an array
341
+ of strings or symbols to listen to multiple application topics with a single
342
+ configuration line.
343
+
344
+ ```ruby
345
+ Rimless.consumer.topics(
346
+ { app: :your_app, names: %i[a b c] } => CustomConsumer
347
+ ).boot!
348
+
349
+ # is identical to:
350
+
351
+ Rimless.consumer.topics(
352
+ { app: :your_app, name: :a } => CustomConsumer,
353
+ { app: :your_app, name: :b } => CustomConsumer,
354
+ { app: :your_app, name: :c } => CustomConsumer
355
+ ).boot!
356
+ ```
357
+
358
+ #### Consuming event messages
359
+
360
+ By convention it makes sense to produce messages with various event types on a
361
+ single Apache Kafka topic. This is fine, they just must follow a single
362
+ constrain: each message must contain an `event`-named field at the Apache Avro
363
+ schema with a dedicated name. This allow to structure data at Kafka like this:
364
+
365
+ ```
366
+ Topic: production.users-api.users
367
+ Events: user_created, user_updated, user_deleted
368
+ ```
369
+
370
+ While respecting this convention your consumer classes will be super clean. See
371
+ the following example: (we keep the users api example)
372
+
373
+ ```ruby
374
+ class UserApiConsumer < ApplicationConsumer
375
+ def user_created(schema_field1:, optional_schema_field2: nil)
376
+ # Do whatever you need when a user was created
377
+ end
378
+ end
379
+ ```
380
+
381
+ Just name a method like the name of the event and specify all Apache Avro
382
+ schema fields of it, except the event field. The messages will be automatically
383
+ decoded with the help of the schema registry. All hashes/arrays ship deeply
384
+ symbolized keys for easy access.
385
+
386
+ **Heads up!** All messages with events which are not reflected by a method will
387
+ just be ignored.
388
+
389
+ See the automatically generated spec (`spec/consumers/custom_consumer_spec.rb`)
390
+ for an example on how to test this.
391
+
392
+ #### Listing consumer routes
393
+
394
+ The rimless gem ships a simple tool to view all your consumer routes and the
395
+ event messages it reacts on. Just run:
396
+
397
+ ```shell
398
+ # Print all Apache Kafka consumer routes
399
+ $ rake rimless:routes
400
+
401
+ # Topic: users-api.users
402
+ # Consumer: UserApiConsumer
403
+ # Events: user_created
404
+ ```
405
+
406
+ #### Starting the consumer process(es)
407
+
408
+ From system integration perspective you just need to start the consumer
409
+ processes and Sidekiq to get the thing going. Rimless allows you to start the
410
+ consumer with `$ rake rimless:consumer` or you can just use the [Karafka
411
+ binary](https://github.com/karafka/karafka/wiki/Fetching-messages) to start the
412
+ consumer (`$ bundle exec karafka server`). Both work identically.
413
+
414
+ When running inside a Rails application the consumer application initialization
415
+ is automatically done for Sidekiq. Otherwise you need to initialize the
416
+ consumer application manually with:
417
+
418
+ ```ruby
419
+ # Manual consumer application initialization
420
+ Sidekiq.configure_server { Rimless.consumer.initialize! }
421
+ ```
422
+
282
423
  ### Encoding/Decoding messages
283
424
 
284
425
  By convention we focus on the [Apache Avro](https://avro.apache.org/) data
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activesupport', '~> 4.2.11'
8
- gem 'railties', '~> 4.2.11'
5
+ gem "activesupport", "~> 4.2.11"
6
+ gem "railties", "~> 4.2.11"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activesupport', '~> 5.0.7'
8
- gem 'railties', '~> 5.0.7'
5
+ gem "activesupport", "~> 5.0.7"
6
+ gem "railties", "~> 5.0.7"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activesupport', '~> 5.1.6'
8
- gem 'railties', '~> 5.1.6'
5
+ gem "activesupport", "~> 5.1.6"
6
+ gem "railties", "~> 5.1.6"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -1,10 +1,8 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
4
2
 
5
- source 'https://rubygems.org'
3
+ source "https://rubygems.org"
6
4
 
7
- gem 'activesupport', '~> 5.2.2'
8
- gem 'railties', '~> 5.2.2'
5
+ gem "activesupport", "~> 5.2.2"
6
+ gem "railties", "~> 5.2.2"
9
7
 
10
- gemspec path: '../'
8
+ gemspec path: "../"
@@ -0,0 +1,8 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "activesupport", "~> 6.0.0"
6
+ gem "railties", "~> 6.0.0"
7
+
8
+ gemspec path: "../"
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rimless
4
+ # The base consumer where all Apache Kafka messages will be processed. It
5
+ # comes with some simple conventions to keep the actual application code
6
+ # simple to use.
7
+ class BaseConsumer < ::Karafka::BaseConsumer
8
+ # A generic message consuming handler which resolves the message event name
9
+ # to an actual method. All message data (top-level keys) is passed down to
10
+ # the event method as symbol arguments.
11
+ def consume
12
+ # We ignore events we do not handle by definition
13
+ send(event, **arguments) if respond_to? event
14
+ end
15
+
16
+ # Prepare the message payload as event method arguments.
17
+ #
18
+ # @return [Hash{Symbol => Mixed}] the event method arguments
19
+ def arguments
20
+ params.payload.except(:event)
21
+ end
22
+
23
+ # A shortcut to fetch the event name from the Kafka message.
24
+ #
25
+ # @return [Symbol] the event name of the current message
26
+ def event
27
+ params.payload[:event].to_sym
28
+ end
29
+ end
30
+ end
@@ -53,5 +53,21 @@ module Rimless
53
53
  ENV.fetch('KAFKA_SCHEMA_REGISTRY_URL',
54
54
  'http://schema-registry.message-bus.local')
55
55
  end
56
+
57
+ # The Sidekiq job queue to use for consuming jobs
58
+ config_accessor(:consumer_job_queue) do
59
+ ENV.fetch('KAFKA_SIDEKIQ_JOB_QUEUE', 'default').to_sym
60
+ end
61
+
62
+ # A custom writer for the consumer job queue name.
63
+ #
64
+ # @param val [String, Symbol] the new job queue name
65
+ def consumer_job_queue=(val)
66
+ config.consumer_job_queue = val.to_sym
67
+ # Refresh the consumer job queue
68
+ Rimless::ConsumerJob.sidekiq_options(
69
+ queue: Rimless.configuration.consumer_job_queue
70
+ )
71
+ end
56
72
  end
57
73
  end
@@ -0,0 +1,189 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rimless
4
+ # The global rimless Apache Kafka consumer application based on
5
+ # the Karafka framework.
6
+ #
7
+ # rubocop:disable Style/ClassVars because we just work as a singleton
8
+ class ConsumerApp < ::Karafka::App
9
+ # We track our own initialization with this class variable
10
+ @@rimless_initialized = false
11
+
12
+ class << self
13
+ # Initialize the Karafka framework and our global consumer application
14
+ # with all our conventions/opinions.
15
+ #
16
+ # @return [Rimless::ConsumerApp] our self for chaining
17
+ def initialize!
18
+ # When already initialized, skip it
19
+ return self if @@rimless_initialized
20
+
21
+ # Initialize all the parts one by one
22
+ initialize_rails!
23
+ initialize_monitors!
24
+ initialize_karafka!
25
+ initialize_logger!
26
+ initialize_code_reload!
27
+
28
+ # Load the custom Karafka boot file when it exists, it contains
29
+ # custom configurations and the topic/consumer routing table
30
+ require ::Karafka.boot_file if ::Karafka.boot_file.exist?
31
+
32
+ # Set our custom initialization process as completed to
33
+ # skip subsequent calls
34
+ @@rimless_initialized = true
35
+ self
36
+ end
37
+
38
+ # Check if Rails is available and not already initialized, then
39
+ # initialize it.
40
+ def initialize_rails!
41
+ rails_env = ::Karafka.root.join('config', 'environment.rb')
42
+
43
+ # Stop, when Rails is already initialized
44
+ return if defined? Rails
45
+
46
+ # Stop, when there is no Rails at all
47
+ return unless rails_env.exist?
48
+
49
+ ENV['RAILS_ENV'] ||= 'development'
50
+ ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
51
+ require rails_env
52
+ Rails.application.eager_load!
53
+ end
54
+
55
+ # We like to listen to instrumentation and logging events to allow our
56
+ # users to handle them like they need.
57
+ def initialize_monitors!
58
+ [
59
+ WaterDrop::Instrumentation::StdoutListener.new,
60
+ ::Karafka::Instrumentation::StdoutListener.new,
61
+ ::Karafka::Instrumentation::ProctitleListener.new
62
+ ].each do |listener|
63
+ ::Karafka.monitor.subscribe(listener)
64
+ end
65
+ end
66
+
67
+ # Configure the pure basics on the Karafka application.
68
+ #
69
+ # rubocop:disable Metrics/MethodLength because of the various settings
70
+ def initialize_karafka!
71
+ setup do |config|
72
+ mapper = Rimless::Karafka::PassthroughMapper.new
73
+ config.consumer_mapper = config.topic_mapper = mapper
74
+ config.deserializer = Rimless::Karafka::AvroDeserializer.new
75
+ config.kafka.seed_brokers = Rimless.configuration.kafka_brokers
76
+ config.client_id = Rimless.configuration.client_id
77
+ config.logger = Rimless.logger
78
+ config.backend = :sidekiq
79
+ config.batch_fetching = true
80
+ config.batch_consuming = false
81
+ config.shutdown_timeout = 10
82
+ end
83
+ end
84
+ # rubocop:enable Metrics/MethodLength
85
+
86
+ # When we run in development mode, we want to write the logs
87
+ # to file and to stdout.
88
+ def initialize_logger!
89
+ return unless Rimless.env.development? && server?
90
+
91
+ STDOUT.sync = true
92
+ Rimless.logger.extend(ActiveSupport::Logger.broadcast(
93
+ ActiveSupport::Logger.new($stdout)
94
+ ))
95
+ end
96
+
97
+ # Perform code hot-reloading when we are in Rails and in development
98
+ # mode.
99
+ def initialize_code_reload!
100
+ return unless defined?(Rails) && Rails.env.development?
101
+
102
+ ::Karafka.monitor.subscribe(::Karafka::CodeReloader.new(
103
+ *Rails.application.reloaders
104
+ ))
105
+ end
106
+
107
+ # Allows the user to re-configure the Karafka application if this is
108
+ # needed. (eg. to set some ruby-kafka driver default settings, etc)
109
+ #
110
+ # @return [Rimless::ConsumerApp] our self for chaining
111
+ def configure(&block)
112
+ setup(&block)
113
+ self
114
+ end
115
+
116
+ # Configure the topics-consumer routing table in a lean way.
117
+ #
118
+ # Examples:
119
+ #
120
+ # topics({ app: :test_app, name: :admins } => YourConsumer)
121
+ # topics({ app: :test_app, names: %i[users admins] } => YourConsumer)
122
+ #
123
+ # @param topics [Hash{Hash => Class}] the topic to consumer mapping
124
+ #
125
+ # rubocop:disable Metrics/MethodLength because of the Karafka DSL
126
+ def topics(topics)
127
+ consumer_groups.draw do
128
+ consumer_group Rimless.configuration.client_id do
129
+ topics.each do |topic_parts, dest_consumer|
130
+ Rimless.consumer.topic_names(topic_parts).each do |topic_name|
131
+ topic(topic_name) do
132
+ consumer dest_consumer
133
+ worker Rimless::ConsumerJob
134
+ interchanger Rimless::Karafka::Base64Interchanger
135
+ end
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ self
142
+ end
143
+ # rubocop:enable Metrics/MethodLength
144
+
145
+ # Build the conventional Apache Kafka topic names from the given parts.
146
+ # This allows various forms like single strings/symbols and a hash in the
147
+ # form of +{ app: [String, Symbol], name: [String, Symbol], names:
148
+ # [Array<String, Symbol>] }+. This allows the maximum of flexibility.
149
+ #
150
+ # @param parts [String, Symbol, Hash{Symbol => Mixed}] the topic
151
+ # name parts
152
+ # @return [Array<String>] the full topic names
153
+ def topic_names(parts)
154
+ # We have a single app, but multiple names so we handle them
155
+ if parts.is_a?(Hash) && parts.key?(:names)
156
+ return parts[:names].map do |name|
157
+ Rimless.topic(parts.merge(name: name))
158
+ end
159
+ end
160
+
161
+ # We cannot handle the given input
162
+ [Rimless.topic(parts)]
163
+ end
164
+
165
+ # Check if we run as the Karafka server (consumer) process or not.
166
+ #
167
+ # @return [Boolean] whenever we run as the Karafka server or not
168
+ def server?
169
+ $PROGRAM_NAME.end_with?('karafka') && ARGV.include?('server')
170
+ end
171
+ end
172
+ end
173
+ # rubocop:enable Style/ClassVars
174
+
175
+ # A rimless top-level concern which adds lean access to
176
+ # the consumer application.
177
+ module Consumer
178
+ extend ActiveSupport::Concern
179
+
180
+ class_methods do
181
+ # A simple shortcut to fetch the Karafka consumer application.
182
+ #
183
+ # @return [Rimless::ConsumerApp] the Karafka consumer application class
184
+ def consumer
185
+ ConsumerApp.initialize!
186
+ end
187
+ end
188
+ end
189
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rimless
4
+ # The base consumer job where each message is processed asynchronous via
5
+ # Sidekiq. We need to inherit the Karafka base worker class into a custom
6
+ # one, otherwise it fails.
7
+ class ConsumerJob < ::Karafka::BaseWorker
8
+ sidekiq_options queue: Rimless.configuration.consumer_job_queue
9
+ end
10
+ end
@@ -35,7 +35,7 @@ module Rimless
35
35
 
36
36
  raise ArgumentError, 'No name given' if name.nil?
37
37
 
38
- "#{Rimless.topic_prefix(app)}#{name}"
38
+ "#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
39
39
  end
40
40
  # rubocop:enable Metrics/AbcSize
41
41
 
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rimless
4
+ module Karafka
5
+ # A custom Apache Avro compatible message deserializer.
6
+ class AvroDeserializer
7
+ # Deserialize an Apache Avro encoded Apache Kafka message.
8
+ #
9
+ # @param message [String] the binary blob to deserialize
10
+ # @return [Hash{Symbol => Mixed}] the deserialized Apache Avro message
11
+ def call(message)
12
+ # We use sparsed hashes inside of Apache Avro messages for schema-less
13
+ # blobs of data, such as loosely structured metadata blobs. Thats a
14
+ # somewhat bad idea on strictly typed and defined messages, but their
15
+ # occurence should be rare.
16
+ Rimless
17
+ .decode(message.payload)
18
+ .yield_self { |data| Sparsify(data, sparse_array: true) }
19
+ .yield_self { |data| data.transform_keys { |key| key.delete('\\') } }
20
+ .yield_self { |data| Unsparsify(data, sparse_array: true) }
21
+ .deep_symbolize_keys
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rimless
4
+ module Karafka
5
+ # Allow the +karafka-sidekiq-backend+ gem to transfer binary Apache Kafka
6
+ # messages to the actual Sidekiq job.
7
+ #
8
+ # rubocop:disable Security/MarshalLoad because we encode/decode the
9
+ # messages in our own controlled context
10
+ class Base64Interchanger
11
+ # Encode a binary Apache Kafka message(s) so they can be passed to the
12
+ # Sidekiq +Rimless::ConsumerJob+.
13
+ #
14
+ # @param params_batch [Mixed] the raw message(s) to encode
15
+ # @return [String] the marshaled+base64 encoded data
16
+ def self.encode(params_batch)
17
+ Base64.encode64(Marshal.dump(params_batch.to_a))
18
+ end
19
+
20
+ # Decode the binary Apache Kafka message(s) so they can be processed by
21
+ # the Sidekiq +Rimless::ConsumerJob+.
22
+ #
23
+ # @param params_string [String] the marshaled+base64 encoded data
24
+ # @return [Mixed] the unmarshaled+base64 decoded data
25
+ def self.decode(params_string)
26
+ Marshal.load(Base64.decode64(params_string))
27
+ end
28
+ end
29
+ # rubocop:enable Security/MarshalLoad
30
+ end
31
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rimless
4
+ module Karafka
5
+ # The Karafka framework makes some assumptions about the consumer group and
6
+ # topic names. We have our own opinions/conventions, so we just pass them
7
+ # through unmodified.
8
+ class PassthroughMapper
9
+ # We do not want to modify the given consumer group name, so we
10
+ # pass it through.
11
+ #
12
+ # @param raw_consumer_group_name [String, Symbol] the original
13
+ # consumer group name
14
+ # @return [String, Symbol] the original consumer group name
15
+ def call(raw_consumer_group_name)
16
+ raw_consumer_group_name
17
+ end
18
+
19
+ # We do not want to modify the given topic name, so we pass it through.
20
+ #
21
+ # @param topic [String, Symbol] the original topic name
22
+ # @return [String, Symbol] the original topic name
23
+ def incoming(topic)
24
+ topic
25
+ end
26
+ alias outgoing incoming
27
+ end
28
+ end
29
+ end
@@ -20,6 +20,18 @@ module Rimless
20
20
  config.after_initialize do
21
21
  # Reconfigure our dependencies
22
22
  Rimless.configure_dependencies
23
+
24
+ # Load the Karafka application inside the Sidekiq server application
25
+ if defined? Sidekiq
26
+ Sidekiq.configure_server do
27
+ Rimless.consumer.initialize!
28
+ end
29
+ end
30
+ end
31
+
32
+ # Load all our Rake tasks if we're supposed to do
33
+ rake_tasks do
34
+ Dir[File.join(__dir__, 'tasks', '*.rake')].each { |file| load file }
23
35
  end
24
36
  end
25
37
  end
@@ -13,6 +13,31 @@ module Rimless
13
13
  def avro_parse(data, **opts)
14
14
  Rimless.avro_decode(data, **opts)
15
15
  end
16
+
17
+ # A simple helper to fake a deserialized Apache Kafka message for
18
+ # consuming.
19
+ #
20
+ # @param payload [Hash{Symbol => Mixed}] the message payload
21
+ # @param topic [String, Hash{Symbol => Mixed}] the actual message
22
+ # topic (full as string, or parts via hash)
23
+ # @return [OpenStruct] the fake deserialized Kafka message
24
+ #
25
+ # rubocop:disable Metrics/MethodLength because of the various properties
26
+ def kafka_message(topic: nil, headers: {}, **payload)
27
+ OpenStruct.new(
28
+ topic: Rimless.topic(topic),
29
+ headers: headers,
30
+ payload: payload,
31
+ is_control_record: false,
32
+ key: nil,
33
+ offset: 206,
34
+ partition: 0,
35
+ create_time: Time.current,
36
+ receive_time: Time.current,
37
+ deserialized: true
38
+ )
39
+ end
40
+ # rubocop:enable Metrics/MethodLength
16
41
  end
17
42
  end
18
43
  end
data/lib/rimless/rspec.rb CHANGED
@@ -6,6 +6,7 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
6
6
  require 'rimless'
7
7
  require 'rimless/rspec/helpers'
8
8
  require 'rimless/rspec/matchers'
9
+ require 'karafka/testing/rspec/helpers'
9
10
 
10
11
  # RSpec 1.x and 2.x compatibility
11
12
  #
@@ -13,10 +14,17 @@ require 'rimless/rspec/matchers'
13
14
  raise 'No RSPEC_CONFIGURER is defined, webmock is missing?' \
14
15
  unless defined?(RSPEC_CONFIGURER)
15
16
 
17
+ # rubocop:disable Metrics/BlockLength because we have to configure
18
+ # RSpec properly
16
19
  RSPEC_CONFIGURER.configure do |config|
17
20
  config.include Rimless::RSpec::Helpers
18
21
  config.include Rimless::RSpec::Matchers
19
22
 
23
+ # Set the custom +consumer+ type for consumer spec files
24
+ config.define_derived_metadata(file_path: %r{/spec/consumers/}) do |meta|
25
+ meta[:type] = :consumer
26
+ end
27
+
20
28
  # Take care of the initial test configuration.
21
29
  config.before(:suite) do
22
30
  # This allows parallel test execution without race conditions on the
@@ -37,7 +45,7 @@ RSPEC_CONFIGURER.configure do |config|
37
45
  # the help of the faked (inlined) Schema Registry server. This allows us to
38
46
  # perform the actual Apache Avro message encoding/decoding without the need
39
47
  # to have a Schema Registry up and running.
40
- config.before(:each) do
48
+ config.before(:each) do |example|
41
49
  # Get the Excon connection from the AvroTurf instance
42
50
  connection = Rimless.avro.instance_variable_get(:@registry)
43
51
  .instance_variable_get(:@upstream)
@@ -55,5 +63,10 @@ RSPEC_CONFIGURER.configure do |config|
55
63
 
56
64
  # Reconfigure the Rimless AvroTurf instance
57
65
  Rimless.configure_avro_turf
66
+
67
+ # When the example type is a Kafka consumer, we must initialize
68
+ # the Karafka framework first.
69
+ Rimless.consumer.initialize! if example.metadata[:type] == :consumer
58
70
  end
59
71
  end
72
+ # rubocop:enable Metrics/BlockLength
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ namespace :rimless do
4
+ desc 'Start the Apache Kafka consumer'
5
+ task :consumer do
6
+ system 'bundle exec karafka server'
7
+ end
8
+
9
+ desc 'Print all the consumer routes'
10
+ task routes: :environment do
11
+ require 'rimless'
12
+
13
+ Rimless.consumer.consumer_groups.each do |consumer_group|
14
+ consumer_group.topics.each do |topic|
15
+ name = topic.name.split('.')[1..-1].join('.')
16
+
17
+ puts "# Topic: #{name}"
18
+ puts "# Consumer: #{topic.consumer}"
19
+
20
+ base = topic.consumer.superclass.new(topic).methods
21
+ events = topic.consumer.new(topic).methods - base
22
+
23
+ puts "# Events: #{events.join(', ')}"
24
+ puts
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ namespace :rimless do
4
+ require 'fileutils'
5
+
6
+ # Install a template file to the project.
7
+ #
8
+ # @param src [String] the template source file name
9
+ # @param dest [Array<String>] the relative destination parts
10
+ def install_template(src, *dest)
11
+ src = File.join(__dir__, 'templates', src)
12
+ dest = File.join(Dir.pwd, *dest, File.basename(src))
13
+
14
+ return puts "# [Skip] #{dest}" if File.exist? dest
15
+
16
+ puts "# [Install] #{dest}"
17
+ FileUtils.mkdir_p(File.dirname(dest))
18
+ FileUtils.copy(src, dest)
19
+ end
20
+
21
+ desc 'Install the Rimless consumer components'
22
+ task :install do
23
+ install_template('karafka.rb')
24
+ install_template('application_consumer.rb', 'app', 'consumers')
25
+ install_template('custom_consumer.rb', 'app', 'consumers')
26
+ install_template('custom_consumer_spec.rb', 'spec', 'consumers')
27
+
28
+ puts <<~OUTPUT
29
+ #
30
+ # Installation done.
31
+ #
32
+ # You can now configure your routes at the +karafka.rb+ file at
33
+ # your project root. And list all routes with +rake rimless:routes+.
34
+ OUTPUT
35
+ end
36
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ if defined?(Rails) && !Rails.env.production?
4
+ require 'rspec/core/rake_task'
5
+
6
+ task :stats do
7
+ require 'rails/code_statistics'
8
+
9
+ [
10
+ [:unshift, 'Consumer', 'app/consumers']
11
+ ].each do |method, type, dir|
12
+ ::STATS_DIRECTORIES.send(method, [type, dir])
13
+ ::CodeStatistics::TEST_TYPES << type if type.include? 'specs'
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ # The overall shared base consumer for Apache Kafka messages. Just write your
4
+ # own specific consumer and inherit this one to share logic.
5
+ class ApplicationConsumer < Rimless::BaseConsumer
6
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ # A dedicated consumer to handle event-messages from your producing application.
4
+ # Just write a method with the name of an event and it is called directly with
5
+ # all the event data as parameters.
6
+ class CustomConsumer < ApplicationConsumer
7
+ # Handle +custom_event+ event messages.
8
+ def custom_event(property1:, property2: nil)
9
+ # Do whatever you need to do
10
+ [property1, property2]
11
+ end
12
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails_helper'
4
+
5
+ RSpec.describe CustomConsumer do
6
+ let(:topic) { Rimless.topic(app: :your_app, name: :your_topic) }
7
+ let(:instance) { karafka_consumer_for(topic) }
8
+ let(:action) { instance.consume }
9
+ let(:params) { kafka_message(topic: topic, **payload) }
10
+
11
+ before { allow(instance).to receive(:params).and_return(params) }
12
+
13
+ context 'with custom_event message' do
14
+ let(:payload) do
15
+ { event: 'custom_event', property1: 'test', property2: nil }
16
+ end
17
+
18
+ it 'returns the payload properties' do
19
+ expect(action).to be_eql(['test', nil])
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rimless'
4
+
5
+ # Setup the topic-consumer routing table and boot the consumer application
6
+ Rimless.consumer.topics(
7
+ { app: :your_app, name: :your_topic } => CustomConsumer
8
+ ).boot!
9
+
10
+ # Configure Karafka/ruby-kafka settings
11
+ # Rimless.consumer.configure do |config|
12
+ # # See https://github.com/karafka/karafka/wiki/Configuration
13
+ # # config.kafka.start_from_beginning = false
14
+ # end
15
+
16
+ # We want a less verbose logging on development
17
+ # Rimless.logger.level = Logger::INFO if Rails.env.development?
18
+
19
+ # Use a different Sidekiq queue for the consumer jobs
20
+ # Rimless.configuration.consumer_job_queue = :messages
@@ -2,5 +2,5 @@
2
2
 
3
3
  module Rimless
4
4
  # The version of the +rimless+ gem
5
- VERSION = '0.3.0'
5
+ VERSION = '1.0.0'
6
6
  end
data/lib/rimless.rb CHANGED
@@ -11,6 +11,8 @@ require 'active_support/core_ext/hash'
11
11
  require 'active_support/core_ext/string'
12
12
  require 'waterdrop'
13
13
  require 'avro_turf/messaging'
14
+ require 'karafka'
15
+ require 'karafka-sidekiq-backend'
14
16
  require 'sparsify'
15
17
  require 'erb'
16
18
  require 'pp'
@@ -24,6 +26,16 @@ module Rimless
24
26
  autoload :AvroUtils, 'rimless/avro_utils'
25
27
  autoload :KafkaHelpers, 'rimless/kafka_helpers'
26
28
  autoload :Dependencies, 'rimless/dependencies'
29
+ autoload :BaseConsumer, 'rimless/base_consumer'
30
+ autoload :Consumer, 'rimless/consumer'
31
+ autoload :ConsumerJob, 'rimless/consumer_job'
32
+
33
+ # All Karafka-framework related components
34
+ module Karafka
35
+ autoload :Base64Interchanger, 'rimless/karafka/base64_interchanger'
36
+ autoload :PassthroughMapper, 'rimless/karafka/passthrough_mapper'
37
+ autoload :AvroDeserializer, 'rimless/karafka/avro_deserializer'
38
+ end
27
39
 
28
40
  # Load standalone code
29
41
  require 'rimless/version'
@@ -34,4 +46,5 @@ module Rimless
34
46
  include Rimless::AvroHelpers
35
47
  include Rimless::KafkaHelpers
36
48
  include Rimless::Dependencies
49
+ include Rimless::Consumer
37
50
  end
File without changes
data/rimless.gemspec CHANGED
@@ -25,6 +25,9 @@ Gem::Specification.new do |spec|
25
25
 
26
26
  spec.add_runtime_dependency 'activesupport', '>= 4.2.0'
27
27
  spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
28
+ spec.add_runtime_dependency 'karafka', '~> 1.3'
29
+ spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.3'
30
+ spec.add_runtime_dependency 'karafka-testing', '~> 1.3'
28
31
  spec.add_runtime_dependency 'sinatra'
29
32
  spec.add_runtime_dependency 'sparsify', '~> 1.1'
30
33
  spec.add_runtime_dependency 'waterdrop', '~> 1.2'
@@ -34,7 +37,7 @@ Gem::Specification.new do |spec|
34
37
  spec.add_development_dependency 'bundler', '>= 1.16', '< 3'
35
38
  spec.add_development_dependency 'factory_bot', '~> 4.11'
36
39
  spec.add_development_dependency 'railties', '>= 4.2.0'
37
- spec.add_development_dependency 'rake', '~> 10.0'
40
+ spec.add_development_dependency 'rake', '~> 13.0'
38
41
  spec.add_development_dependency 'rdoc', '~> 6.1'
39
42
  spec.add_development_dependency 'redcarpet', '~> 3.4'
40
43
  spec.add_development_dependency 'rspec', '~> 3.0'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rimless
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hermann Mayer
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-11-08 00:00:00.000000000 Z
11
+ date: 2020-03-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -38,6 +38,48 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: 0.11.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: karafka
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.3'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.3'
55
+ - !ruby/object:Gem::Dependency
56
+ name: karafka-sidekiq-backend
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.3'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.3'
69
+ - !ruby/object:Gem::Dependency
70
+ name: karafka-testing
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1.3'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1.3'
41
83
  - !ruby/object:Gem::Dependency
42
84
  name: sinatra
43
85
  requirement: !ruby/object:Gem::Requirement
@@ -162,14 +204,14 @@ dependencies:
162
204
  requirements:
163
205
  - - "~>"
164
206
  - !ruby/object:Gem::Version
165
- version: '10.0'
207
+ version: '13.0'
166
208
  type: :development
167
209
  prerelease: false
168
210
  version_requirements: !ruby/object:Gem::Requirement
169
211
  requirements:
170
212
  - - "~>"
171
213
  - !ruby/object:Gem::Version
172
- version: '10.0'
214
+ version: '13.0'
173
215
  - !ruby/object:Gem::Dependency
174
216
  name: rdoc
175
217
  requirement: !ruby/object:Gem::Requirement
@@ -345,18 +387,33 @@ files:
345
387
  - gemfiles/rails_5.0.gemfile
346
388
  - gemfiles/rails_5.1.gemfile
347
389
  - gemfiles/rails_5.2.gemfile
390
+ - gemfiles/rails_6.0.gemfile
348
391
  - lib/rimless.rb
349
392
  - lib/rimless/avro_helpers.rb
350
393
  - lib/rimless/avro_utils.rb
394
+ - lib/rimless/base_consumer.rb
351
395
  - lib/rimless/configuration.rb
352
396
  - lib/rimless/configuration_handling.rb
397
+ - lib/rimless/consumer.rb
398
+ - lib/rimless/consumer_job.rb
353
399
  - lib/rimless/dependencies.rb
354
400
  - lib/rimless/kafka_helpers.rb
401
+ - lib/rimless/karafka/avro_deserializer.rb
402
+ - lib/rimless/karafka/base64_interchanger.rb
403
+ - lib/rimless/karafka/passthrough_mapper.rb
355
404
  - lib/rimless/railtie.rb
356
405
  - lib/rimless/rspec.rb
357
406
  - lib/rimless/rspec/helpers.rb
358
407
  - lib/rimless/rspec/matchers.rb
408
+ - lib/rimless/tasks/consumer.rake
409
+ - lib/rimless/tasks/generator.rake
410
+ - lib/rimless/tasks/stats.rake
411
+ - lib/rimless/tasks/templates/application_consumer.rb
412
+ - lib/rimless/tasks/templates/custom_consumer.rb
413
+ - lib/rimless/tasks/templates/custom_consumer_spec.rb
414
+ - lib/rimless/tasks/templates/karafka.rb
359
415
  - lib/rimless/version.rb
416
+ - log/development.log
360
417
  - rimless.gemspec
361
418
  homepage: https://github.com/hausgold/rimless
362
419
  licenses: []
@@ -376,7 +433,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
376
433
  - !ruby/object:Gem::Version
377
434
  version: '0'
378
435
  requirements: []
379
- rubygems_version: 3.0.6
436
+ rubygems_version: 3.1.2
380
437
  signing_key:
381
438
  specification_version: 4
382
439
  summary: A bundle of opinionated Apache Kafka / Confluent Schema Registry helpers.