capistrano-karafka 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 369db3566eeacd1e9a68a38f222fadd990856a74
-  data.tar.gz: 294d4ac5bf51d1d87a760b6547335c300a3686b8
+  metadata.gz: 4c508336eb91f1a79204656633d7886e7d2c6581
+  data.tar.gz: 763280f3e84d79226238bc9b0a77a0bb9b96c26d
 SHA512:
-  metadata.gz: 2033aa8fa0476cf716c31a2917ff2ab7e32b4d33cd7b959e0abd4a6c1feb345e71eda6221a761484709da52a2332beda6cd52e5a367165344d482cf4e368c120
-  data.tar.gz: 5741accff7643074cf38f844df021f0368e1ae622c49fc378dda8e7ebd06b36f8b357451632bff500d5cfdd4168fb6c782697c85c49f8b46cf136b94315f5a7e
+  metadata.gz: c07f9c784e1ca4baf4b22187dd79d334e1c22aee46a7d88b2800e93d0b24141592a11f603cd5036654bbf816f4d3cf0995573b902fef262291369ace35c8066d
+  data.tar.gz: 46c156740fae8bbe3561832e8d2a86cae2b616b104eebe02cb7ef6e94d9d6948f77f1ef1ccc3d71796f35b69734a34f55a3c58e42fab7af981b67d8eb54aab35

data/.gitignore

@@ -4,7 +4,6 @@
 .config
 .idea
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Capistrano Karafka changelog
 
+## 1.1.0
+- Support of Karafka 0.6+
+- Support of multiple Karafka processes execution
+- Support of multiple Karafka servers deployment
+- Support for specifying customer groups, including multi-role customer groups execution
+
 ## 1.0.0
 
 - Gem init

data/Gemfile.lock

@@ -0,0 +1,133 @@
+PATH
+  remote: .
+  specs:
+    capistrano-karafka (1.1.0)
+      capistrano (>= 3.9)
+      capistrano-bundler
+      karafka (>= 0.5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.1.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    airbrussh (1.3.0)
+      sshkit (>= 1.6.1, != 1.7.0)
+    capistrano (3.9.0)
+      airbrussh (>= 1.0.0)
+      i18n
+      rake (>= 10.0.0)
+      sshkit (>= 1.9.0)
+    capistrano-bundler (1.2.0)
+      capistrano (~> 3.1)
+      sshkit (~> 1.2)
+    celluloid (0.17.3)
+      celluloid-essentials
+      celluloid-extras
+      celluloid-fsm
+      celluloid-pool
+      celluloid-supervision
+      timers (>= 4.1.1)
+    celluloid-essentials (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-extras (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-fsm (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-pool (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-supervision (0.20.6)
+      timers (>= 4.1.1)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
+    dry-configurable (0.7.0)
+      concurrent-ruby (~> 1.0)
+    dry-container (0.6.0)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+    dry-core (0.3.1)
+      concurrent-ruby (~> 1.0)
+    dry-equalizer (0.2.0)
+    dry-logic (0.4.1)
+      dry-container (~> 0.2, >= 0.2.6)
+      dry-core (~> 0.2)
+      dry-equalizer (~> 0.2)
+    dry-types (0.11.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1)
+      dry-container (~> 0.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.0)
+      inflecto (~> 0.0.0, >= 0.0.2)
+    dry-validation (0.10.7)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.0)
+      dry-types (~> 0.9, >= 0.9.0)
+    envlogic (1.0.3)
+      activesupport
+    hitimes (1.2.6)
+    i18n (0.8.6)
+    inflecto (0.0.2)
+    karafka (0.5.0.3)
+      activesupport (~> 5.0)
+      celluloid (~> 0.17)
+      dry-configurable (~> 0.7)
+      dry-validation (~> 0.10.6)
+      envlogic (~> 1.0)
+      rake (~> 11.3)
+      ruby-kafka (= 0.3.17)
+      sidekiq (>= 4.2)
+      thor (~> 0.19)
+      waterdrop (~> 0.3.2.4)
+      worker-glass (~> 0.2)
+    minitest (5.10.3)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (4.1.0)
+    null-logger (0.1.4)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
+    rake (11.3.0)
+    redis (3.3.3)
+    ruby-kafka (0.3.17)
+    sidekiq (5.0.4)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (~> 3.3, >= 3.3.3)
+    sshkit (1.14.0)
+      net-scp (>= 1.1.2)
+      net-ssh (>= 2.8.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    timers (4.1.2)
+      hitimes
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    waterdrop (0.3.2.4)
+      bundler
+      connection_pool
+      dry-configurable (~> 0.6)
+      null-logger
+      rake
+      ruby-kafka
+    worker-glass (0.2.4)
+      activesupport
+      null-logger
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  capistrano-karafka!
+
+BUNDLED WITH
+   1.15.4

data/README.md

@@ -7,7 +7,7 @@ Karafka integration for Capistrano.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Add this line to your application Gemfile:
 
 ```ruby
   gem 'capistrano-karafka'
@@ -37,16 +37,63 @@ install_plugin Capistrano::Karafka
 Take a look at the [set_defaults](https://github.com/karafka/capistrano-karafka/blob/master/lib/capistrano/karafka.rb#L16) method for options you can set. For example, to specify a different pidfile than default:
 
 ```ruby
-set :karafka_pid, ->{ File.join(shared_path, 'tmp', 'pids', 'karafka0') }
+set :karafka_pid, ->{ File.join(shared_path, 'tmp', 'pids', 'karafka.pid') }
 ```
 
+**Warning**: In order to make capistrano-karafka work with multiple processes, pidfile needs to end with ```.pid```.
+
+## Multiple processes
+
+You can configure Karafka to start with multiple processes. Just set the proper amount in ```karafka_processes```.
+
+```ruby
+set :karafka_processes, 4
+```
+
+## Different number of processes per host
+
+You can configure how many processes you want to run on each host in the following way:
+
+```ruby
+set :karafka_role, %i[karafka_small karafka_big]
+set :karafka_small_processes, 1
+set :karafka_big_processes, 4
+server 'example-small.com', roles: %i[karafka_small]
+server 'example-big.com', roles: %i[karafka_big]
+```
+
+## Different active consumer groups per host and process
+
+You can also easily start Karafka with a limited set of consumer groups running per each process for a given host:
+
+```ruby
+set :karafka_role, %i[karafka_small karafka_big]
+
+set :karafka_small_processes, 1
+set :karafka_small_consumer_groups, %w[
+  group_a
+]
+
+set :karafka_big_processes, 4
+set :karafka_small_consumer_groups, [
+  'group_a group_b',
+  'group_c group_d',
+  'group_e',
+  'group_f'
+]
+
+server 'example-small.com', roles: %i[karafka_small]
+server 'example-big.com', roles: %i[karafka_big]
+```
+
+By default, capistrano-karafka will start Karafka processes without limiting consumer groups, which means that each process will be running all of them.
+
 ## References
 
 * [Karafka framework](https://github.com/karafka/karafka)
 * [Capistrano Karafka](https://github.com/karafka/capistrano-karafka)
 * [Waterdrop](https://github.com/karafka/waterdrop)
 * [Envlogic](https://github.com/karafka/envlogic)
-* [Worker Glass](https://github.com/karafka/worker-glass)
 * [Null Logger](https://github.com/karafka/null-logger)
 
 ## Note on Patches/Pull Requests
@@ -56,6 +103,6 @@ Make your feature addition or bug fix.
 Add tests for it. This is important so I don't break it in a future version unintentionally.
 Commit, do not mess with Rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull). Send me a pull request. Bonus points for topic branches.
 
-[![coditsu](https://coditsu.io/assets/quality_bar.png)](https://coditsu.io)
+[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/coditsu/repositories/capistrano-karafka)
 
-Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combinse multiple linters and code analyzers. Unfortunately, for now it is invite-only based, so just ping us and we will give you access to the quality results.
+Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://app.coditsu.io/coditsu/repositories/capistrano-karafka) that combinse multiple linters and code analyzers.

data/capistrano-karafka.gemspec

@@ -3,11 +3,9 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'capistrano/karafka/version'
-
 Gem::Specification.new do |spec|
   spec.name        = 'capistrano-karafka'
-  spec.version     = Capistrano::Karafka::VERSION
+  spec.version     = '1.1.0'
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[maciej@coditsu.io]
@@ -16,9 +14,9 @@ Gem::Specification.new do |spec|
   spec.description = 'Karafka integration for Capistrano'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'capistrano', '~> 3.8'
+  spec.add_dependency 'capistrano', '>= 3.9'
   spec.add_dependency 'capistrano-bundler'
-  spec.add_dependency 'karafka', '~> 0'
+  spec.add_dependency 'karafka', '>= 0.5'
   spec.required_ruby_version = '>= 2.3.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }

data/lib/capistrano/karafka.rb

@@ -7,14 +7,20 @@ require 'capistrano/plugin'
 module Capistrano
   # Karafka Capistrano integration
   class Karafka < Capistrano::Plugin
-    # Defines all the capistrano tasks by taking them from the rake cap file
-    def define_tasks
-      eval_rakefile File.expand_path('../tasks/karafka.cap', __FILE__)
-    end
+    # Names of files that store capistrano procedures
+    CAP_FILES = %w[
+      karafka
+      karafka/start
+      karafka/stop
+      karafka/restart
+      karafka/status
+    ].freeze
 
     # Default values for Karafka settings
     def set_defaults
-      set_if_empty :karafka_role, :app
+      set_if_empty :karafka_role, :karafka
+      set_if_empty :karafka_processes, 1
+      set_if_empty :karafka_consumer_groups, []
       set_if_empty :karafka_default_hooks, -> { true }
       set_if_empty :karafka_env, -> { fetch(:karafka_env, fetch(:environment)) }
       set_if_empty :karafka_pid, -> { File.join(shared_path, 'tmp', 'pids', 'karafka.pid') }
@@ -26,3 +32,7 @@ module Capistrano
     end
   end
 end
+
+Capistrano::Karafka::CAP_FILES.each do |cap_file|
+  eval File.open(File.expand_path("../tasks/#{cap_file}.cap", __FILE__), 'r').read
+end

data/lib/capistrano/tasks/karafka.cap

@@ -1,67 +1,35 @@
 # frozen_string_literal: true
 
-# @note Inspired by Puma capistrano handlers
+# @note Inspired by Puma capistrano handlers and Sidekiq capistrano handlers
 # @see https://github.com/seuros/capistrano-puma/blob/master/lib/capistrano/tasks/puma.rake
+# @see https://github.com/seuros/capistrano-sidekiq/blob/master/lib/capistrano/tasks/capistrano2.rb
 namespace :karafka do
-  desc 'Stop Karafka'
-  task :stop do
-    on roles(fetch(:karafka_role)) do |_host|
-      within shared_path do
-        # If there's no pidfile it means that Karafka is not running
-        next unless test "cat #{fetch(:karafka_pid)}"
-
-        # Send a kill signal to a given process
-        execute "kill -INT `cat #{fetch(:karafka_pid)}`"
-
-        # And wait until it finishes. We wait because we don't want to start next process until
-        # the previous one is stopped. That way we won't have problems with Kafka registering and
-        # deregistering processes from topics (although nothing bad would happen. It would just
-        # take more time to rebalance)
-        loop do
-          break unless test "cat #{fetch(:karafka_pid)}"
-          info 'Waiting for Karafka to stop'
-          sleep 5
-        end
-      end
-    end
-  end
-
-  desc 'Start Karafka'
-  task :start do
-    on roles(fetch(:karafka_role)) do |_host|
-      within current_path do
-        # We use all 3 because when combined with Sinatra/Rails it will use their parts as well
-        # so we want to set proper env for any of them
-        with(
-          KARAFKA_ENV: fetch(:karafka_env),
-          RAILS_ENV: fetch(:rails_env),
-          RACK_ENV: fetch(:rack_env)
-        ) do
-          execute :bundle, "exec karafka server -d -p #{fetch(:karafka_pid)}"
-        end
+  # @return [Array<Hash>] hash with details about each of the processes that
+  #   we're suppose to manage. Contains pidfile and consumer_groups for now
+  def processes_details
+    processes_details = []
+
+    Array(fetch(:karafka_role)).each do |role|
+      next unless host.roles.include?(role)
+
+      processes = fetch(:"#{role}_processes") || fetch(:karafka_processes)
+      consumer_groups = fetch(:"#{role}_consumer_groups") || fetch(:karafka_consumer_groups)
+
+      processes.times do |index|
+        processes_details << {
+          pidfile: fetch(:karafka_pid).gsub(/\.pid$/, "-#{index}.pid"),
+          consumer_groups: consumer_groups[index]
+        }
       end
     end
-  end
 
-  desc 'Restart Karafka'
-  task :restart do
-    invoke 'karafka:stop'
-    invoke 'karafka:start'
+    processes_details
   end
 
-  desc 'Status Karafka'
-  task :status do
-    on roles(fetch(:karafka_role)) do |_host|
-      if test "cat #{fetch(:karafka_pid)}"
-        pid = capture "cat #{fetch(:karafka_pid)}"
-
-        if test "ps -p #{pid} > /dev/null"
-          info "Karafka is started: #{pid}"
-        else
-          error 'Karafka is not started but pidfile exists'
-        end
-      else
-        info 'Karafka is not started'
+  def for_each_process
+    processes_details.each_with_index do |process_details, index|
+      within release_path do
+        yield(process_details, index)
       end
     end
   end

data/lib/capistrano/tasks/karafka/restart.cap

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+namespace :karafka do
+  desc 'Restart Karafka'
+  task :restart do
+    invoke 'karafka:stop'
+    invoke 'karafka:start'
+  end
+end

data/lib/capistrano/tasks/karafka/start.cap

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+namespace :karafka do
+  # Starts Karafka process with a given set of params
+  # @param [Hash] Details of a process that we're suppose to start
+  def start_karafka(process_details)
+    pidfile = process_details[:pidfile]
+    consumer_groups = process_details[:consumer_groups]
+
+    # We use all 3 because when combined with Sinatra/Rails it will use their parts as well
+    # so we want to set proper env for any of them
+    with(
+      KARAFKA_ENV: fetch(:karafka_env),
+      RAILS_ENV: fetch(:rails_env),
+      RACK_ENV: fetch(:rack_env)
+    ) do
+      execute :bundle, [
+        'exec',
+        'karafka server',
+        '-d',
+        "-p #{pidfile}",
+        consumer_groups ? "-g #{consumer_groups}" : nil
+      ].join(' ')
+    end
+  end
+
+  desc 'Start Karafka'
+  task :start do
+    on roles(fetch(:karafka_role)) do
+      for_each_process do |process_details|
+        start_karafka(process_details)
+      end
+    end
+  end
+end

data/lib/capistrano/tasks/karafka/status.cap

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+namespace :karafka do
+  # Checks a status of a given Karafka process
+  # @param pidfile [String] path to a pidfile of a process that we want to check
+  def status_karafka(pidfile)
+    if test "cat #{pidfile}"
+      process_pid = capture "cat #{pidfile}"
+
+      if test "ps -p #{process_pid} > /dev/null"
+        info "Karafka #{process_pid} is started: #{pidfile}"
+      else
+        error "Karafka is not started but pidfile exists: #{pidfile}"
+      end
+    else
+      info "Karafka is not started: #{pidfile}"
+    end
+  end
+
+  desc 'Status Karafka'
+  task :status do
+    on roles fetch(:karafka_role) do
+      for_each_process do |process_details|
+        status_karafka(process_details[:pidfile])
+      end
+    end
+  end
+end

data/lib/capistrano/tasks/karafka/stop.cap

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+namespace :karafka do
+  # Stops a given Karafka process
+  # @param pidfile [String] path to a pidfile of a process that we're suppose to stop
+  def stop_karafka(pidfile)
+    # If there's no pidfile it means that Karafka is not running
+    unless test "cat #{pidfile}"
+      info "Karafka is not started: #{pidfile}"
+      return
+    end
+
+    # Send a kill signal to a given process
+    execute "kill -INT `cat #{pidfile}`"
+
+    # And wait until it finishes. We wait because we don't want to start next process until
+    # the previous one is stopped. That way we won't have problems with Kafka registering and
+    # deregistering processes from topics (although nothing bad would happen. It would just
+    # take more time to rebalance)
+    loop do
+      break unless test "cat #{pidfile}"
+      info 'Waiting for Karafka to stop'
+      sleep 5
+    end
+  end
+
+  desc 'Stop Karafka'
+  task :stop do
+    on roles(fetch(:karafka_role)) do
+      for_each_process do |process_details|
+        stop_karafka(process_details[:pidfile])
+      end
+    end
+  end
+end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: capistrano-karafka
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Maciej Mensfeld
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-02 00:00:00.000000000 Z
+date: 2017-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capistrano
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: capistrano-bundler
   requirement: !ruby/object:Gem::Requirement
@@ -42,16 +42,16 @@ dependencies:
   name: karafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.5'
 description: Karafka integration for Capistrano
 email:
 - maciej@coditsu.io
@@ -64,14 +64,18 @@ files:
 - ".ruby-version"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.lock
 - MIT-LICENCE
 - README.md
 - Rakefile
 - capistrano-karafka.gemspec
 - lib/capistrano-karafka.rb
 - lib/capistrano/karafka.rb
-- lib/capistrano/karafka/version.rb
 - lib/capistrano/tasks/karafka.cap
+- lib/capistrano/tasks/karafka/restart.cap
+- lib/capistrano/tasks/karafka/start.cap
+- lib/capistrano/tasks/karafka/status.cap
+- lib/capistrano/tasks/karafka/stop.cap
 homepage: https://github.com/karafka/capistrano-karafka
 licenses:
 - MIT

data/lib/capistrano/karafka/version.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-module Capistrano
-  class Karafka
-    # Current capistrano integration gem version
-    VERSION = '1.0.0'
-  end
-end