gitlab-monitor 4.0.0

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+gemspec
+
+group :test do
+  gem "rspec", "~>3.5"
+  gem "rubocop", "~>0.42"
+end

data/Gemfile.lock

@@ -0,0 +1,75 @@
+PATH
+  remote: .
+  specs:
+    gitlab-monitor (4.0.0)
+      connection_pool (~> 2.2.1)
+      pg (~> 1.1)
+      quantile (~> 0.2.0)
+      redis (~> 3.2)
+      redis-namespace (~> 1.6.0)
+      sidekiq (~> 5.2.1)
+      sinatra (~> 2.0.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    connection_pool (2.2.2)
+    diff-lcs (1.3)
+    mustermann (1.0.3)
+    parser (2.5.1.0)
+      ast (~> 2.4.0)
+    pg (1.1.4)
+    powerpack (0.1.1)
+    quantile (0.2.1)
+    rack (2.0.6)
+    rack-protection (2.0.5)
+      rack
+    rainbow (2.1.0)
+    redis (3.3.5)
+    redis-namespace (1.6.0)
+      redis (>= 3.0.4)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rubocop (0.42.0)
+      parser (>= 2.3.1.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.1)
+    sidekiq (5.2.5)
+      connection_pool (~> 2.2, >= 2.2.2)
+      rack (>= 1.5.0)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.5, < 5)
+    sinatra (2.0.5)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.5)
+      tilt (~> 2.0)
+    tilt (2.0.9)
+    unicode-display_width (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  gitlab-monitor!
+  rspec (~> 3.5)
+  rspec-expectations (~> 3.7.0)
+  rubocop (~> 0.42)
+
+BUNDLED WITH
+   1.17.3

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2011-2017 GitLab B.V.
+
+With regard to the GitLab Software:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+For all third party components incorporated into the GitLab Software, those
+components are licensed under the original license provided by the owner of the
+applicable component.

data/README.md

@@ -0,0 +1,110 @@
+## Introduction
+
+gitlab-monitor is a [Prometheus Web exporter] that does the following:
+
+1. Collects GitLab production metrics via custom probes defined in a [YAML
+   configuration file](config/gitlab-monitor.yml.example).
+2. Custom probes gather measurements in the form of key/value pairs.
+3. For each probe, gitlab-monitor creates an HTTP endpoint `/<probe_name>`
+   (by default on port 9168) that delivers these metrics to a Prometheus scraper.
+
+A central Prometheus process is configured to poll exporters at a specified
+frequency.
+
+### Supported Probes
+
+Below is a list of probes added by this exporter, and their corresponding
+metrics.
+
+1. Database
+    * [Per-table tuple stats](lib/gitlab_monitor/database/tuple_stats.rb) --
+      `gitlab_database_stat_table_*`
+    * [Row count queries](lib/gitlab_monitor/database/row_count.rb) --
+      `gitlab_database_rows`
+    * [CI builds](lib/gitlab_monitor/database/ci_builds.rb) --
+      `ci_pending_builds`, `ci_created_builds`, `ci_stale_builds`,
+      `ci_running_builds`
+    * [Bloat](lib/gitlab_monitor/database/bloat.rb) --
+      `gitlab_database_bloat_$type_$key` with type `btree` (index bloat) or `table`
+      (table bloat) and keys `bloat_ratio bloat_size extra_size real_size` (see below)
+    * [Remote mirrors](lib/gitlab_monitor/database/remote_mirrors.rb) --
+      `project_remote_mirror_last_successful_update_time_seconds`, `project_remote_mirror_last_update_time_seconds`
+1. Git
+    * [git pull/push timings](lib/gitlab_monitor/git.rb) --
+      `git_pull_time_milliseconds`, `git_push_time_milliseconds`
+    * git processes stats (see Process below)
+1. [Process](lib/gitlab_monitor/process.rb)
+    * CPU time -- `process_cpu_seconds_total`
+    * Start time -- `process_start_time_seconds`
+    * Count -- `process_count`
+    * Memory usage
+      * Data from /proc/<pid>/cmdline:
+        * `process_resident_memory_bytes`
+        * `process_virtual_memory_bytes`
+      * Data from /proc/<pid>/smaps -- `probe_smaps` (off by default):
+        * `process_smaps_size_bytes`
+        * `process_smaps_rss_bytes`
+        * `process_smaps_shared_clean_bytes`
+        * `process_smaps_shared_dirty_bytes`
+        * `process_smaps_private_clean_bytes`
+        * `process_smaps_private_dirty_bytes`
+        * `process_smaps_swap_bytes`
+        * `process_smaps_pss_bytes`
+1. [Sidekiq](lib/gitlab_monitor/sidekiq.rb) -- `sidekiq_queue_size`, `sidekiq_queue_paused`,
+   `sidekiq_queue_latency_seconds`, `sidekiq_enqueued_jobs`, `sidekiq_dead_jobs`,
+   `sidekiq_running_jobs`, `sidekiq_to_be_retried_jobs`
+
+### Setup with GitLab Development Kit
+
+gitlab-monitor can be setup with the [GitLab Development Kit] for development.
+When using the gitlab-monitor CLI, you'll need to set the `--db-conn` flag to
+connect to the PostgreSQL instance in your GDK folder. For example:
+
+```
+bin/gitlab-mon row-counts --db-conn="dbname=gitlabhq_development host=/Users/<user>/gitlab-development-kit/postgresql"
+```
+
+### Running gitlab-monitor as a Web exporter
+
+When serving the pages on `localhost`, you'll need to edit the YAML
+configuration file. An example can be found under
+[`config/gitlab-monitor.yml.example`](config/gitlab-monitor.yml.example). For
+each probe that has to connect to the database, set the `connection_string` to
+`dbname=gitlabhq_development
+host=/Users/<user>/gitlab-development-kit/postgresql`
+
+Once you have this configured, you can then run:
+
+```
+bin/gitlab-mon web -c config/gitlab-monitor.yml
+```
+
+Once running, you can point your browser or curl to the following URLs:
+
+* http://localhost:9168/database
+* http://localhost:9168/git_process
+* http://localhost:9168/process
+* http://localhost:9168/sidekiq
+* http://localhost:9168/metrics (to get all of the above combined)
+
+### Database Bloat Metrics
+
+Database bloat is measured for indexes (`btree`) and/or tables
+(`table`). Returned metrics contain:
+
+* `bloat_ratio`: estimated ratio of the real size used by bloat_size.
+* `bloat_size`: estimated size of the bloat without the extra space kept for the fillfactor.
+* `extra_size`: estimated extra size not used/needed by the index. This extra size is composed by the fillfactor, bloat and alignment padding spaces.
+* `real_size`: real size of the index
+
+Also see the [original documentation](https://github.com/ioguix/pgsql-bloat-estimation/blob/master/README.md).
+
+Note that all metrics returned are estimates without an upper bound for
+the error.
+
+## Contributing
+
+gitlab-monitor is an open source project and we are very happy to accept community contributions. Please refer to [CONTRIBUTING.md](/CONTRIBUTING.md) for details.
+
+[Prometheus Web exporter]: https://prometheus.io/docs/instrumenting/exporters/
+[GitLab Development Kit]: https://gitlab.com/gitlab-org/gitlab-development-kit

data/bin/gitlab-mon

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+
+require "optparse"
+require "gitlab_monitor"
+
+def main
+  clazz = GitLab::Monitor::CLI.for(ARGV.shift)
+  runner = clazz.new(ARGV)
+  runner.run
+rescue GitLab::Monitor::CLI::InvalidCLICommand => e
+  puts e
+  exit 1
+end
+
+main

data/config/gitlab-monitor.yml.example

@@ -0,0 +1,112 @@
+db_common: &db_common
+  methods:
+    - probe_db
+  opts: &db_common_opts
+    connection_string: dbname=gitlabhq_development user=postgres
+
+# Web server config
+server:
+  listen_address: 0.0.0.0
+  listen_port: 9168
+  # Maximum amount of memory to use in megabytes, after which the process is killed
+  memory_threshold: 1024
+
+# Probes config
+probes:
+  # Each key corresponds to an endpoint, so here metrics are available at http://localhost:9168/git.
+  # The server will search for a prober using the format `KeyProber`, so here it will be `GitProber`.
+  # If there's no prober matching the format above, `class_name` key should be provided (see `git_process` below).
+  git:
+    # Methods to call on the prober
+    methods:
+      - probe_pull
+      - probe_push
+    # Options to pass to the prober class initializer
+    opts:
+      source: /home/git/repo
+
+  git_process: &git_process
+    class_name: GitProcessProber # `class_name` is redundant here
+    methods:
+    - probe_git
+    opts:
+      quantiles: true
+
+  database_bloat:
+    class_name: Database::BloatProber
+    <<: *db_common
+
+  # We can group multiple probes under a single endpoint by setting the `multiple` key to `true`, followed
+  # by probe definitions as usual.
+  database:
+    multiple: true
+    ci_builds:
+      class_name: Database::CiBuildsProber
+      <<: *db_common
+      opts:
+        <<: *db_common_opts
+        allowed_repeated_commands_count: 2
+        created_builds_counting_disabled: true
+        unarchived_traces_offset_minutes: 1440
+    tuple_stats:
+      class_name: Database::TuplesProber
+      <<: *db_common
+    rows_count:
+      class_name: Database::RowCountProber
+      <<: *db_common
+      opts:
+        <<: *db_common_opts
+        selected_queries:
+        - soft_deleted_projects
+        - orphaned_projects
+        - uploads
+    remote_mirrors:
+      class_name: Database::RemoteMirrors
+      <<: *db_common
+      opts:
+        <<: *db_common_opts
+        project_ids:
+        - 1
+
+  process: &process
+    methods:
+      - probe_stat
+      - probe_count
+    opts:
+      - pid_or_pattern: "sidekiq .* \\[.*?\\]"
+        name: sidekiq
+      - pid_or_pattern: "unicorn.* worker\\[.*?\\]"
+        name: unicorn
+      - pid_or_pattern: "git-upload-pack --stateless-rpc"
+        name: git_upload_pack
+        quantiles: true
+
+  sidekiq: &sidekiq
+    methods:
+      - probe_queues
+      - probe_jobs
+      - probe_workers
+      - probe_retries
+      - probe_dead
+    opts:
+      redis_url: "redis://localhost:6379"
+      redis_enable_client: true
+
+  metrics:
+    multiple: true
+    git_process:
+      <<: *git_process
+    process:
+      <<: *process
+    sidekiq:
+      <<: *sidekiq
+    ci_builds:
+      class_name: Database::CiBuildsProber
+      <<: *db_common
+    tuple_stats:
+      class_name: Database::TuplesProber
+      <<: *db_common
+    rows_count:
+      class_name: Database::RowCountProber
+      <<: *db_common
+

data/gitlab-monitor.gemspec

@@ -0,0 +1,33 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "gitlab_monitor/version"
+
+Gem::Specification.new do |s|
+  s.name          = "gitlab-monitor"
+  s.version       = GitLab::Monitor::VERSION
+  s.date          = "2016-07-27"
+  s.summary       = "GitLab monitoring tools"
+  s.description   = "GitLab monitoring tools to use with prometheus"
+  s.authors       = ["Pablo Carranza"]
+  s.email         = "pablo@gitlab.com"
+
+  s.files         = `git ls-files -z`.split("\x0")
+
+  s.executables   = ["gitlab-mon"]
+  s.test_files    = s.files.grep(%r{^(spec)/})
+
+  s.require_paths = ["lib"]
+  s.homepage      = "http://gitlab.com"
+  s.license       = "MIT"
+
+  s.add_runtime_dependency "pg", "~> 1.1"
+  s.add_runtime_dependency "sinatra", "~> 2.0.4"
+  s.add_runtime_dependency "quantile", "~> 0.2.0"
+  s.add_runtime_dependency "sidekiq", "~> 5.2.1"
+  s.add_runtime_dependency "redis", "~> 3.2"
+  s.add_runtime_dependency "redis-namespace", "~> 1.6.0"
+  s.add_runtime_dependency "connection_pool", "~> 2.2.1"
+
+  s.add_development_dependency "rspec", "~> 3.7.0"
+  s.add_development_dependency "rspec-expectations", "~> 3.7.0"
+end

data/lib/gitlab_monitor.rb

@@ -0,0 +1,18 @@
+module GitLab
+  # GitLab Monitoring
+  module Monitor
+    autoload :CLI,               "gitlab_monitor/cli"
+    autoload :TimeTracker,       "gitlab_monitor/util"
+    autoload :Utils,             "gitlab_monitor/util"
+    autoload :PrometheusMetrics, "gitlab_monitor/prometheus"
+    autoload :Utils,             "gitlab_monitor/util"
+    autoload :Git,               "gitlab_monitor/git"
+    autoload :GitProber,         "gitlab_monitor/git"
+    autoload :GitProcessProber,  "gitlab_monitor/git"
+    autoload :Database,          "gitlab_monitor/database"
+    autoload :ProcessProber,     "gitlab_monitor/process"
+    autoload :WebExporter,       "gitlab_monitor/web_exporter"
+    autoload :Prober,            "gitlab_monitor/prober"
+    autoload :SidekiqProber,     "gitlab_monitor/sidekiq"
+  end
+end

data/lib/gitlab_monitor/cli.rb

@@ -0,0 +1,341 @@
+require "yaml"
+
+module GitLab
+  module Monitor
+    # Stores runner classes in a single place
+    #
+    # The entry point is the module method "for" which takes the name of a runner.
+    # In case the runner is invalid it will return a NullRunner which fails with an
+    # InvalidCLICommand error, which contains the general application usage instructions.
+    module CLI
+      EXECUTABLE_NAME = "gitlab-mon".freeze
+
+      def self.for(name)
+        commands.fetch(name, NullRunner)
+      end
+
+      class InvalidCLICommand < RuntimeError; end
+
+      # Empty runner that will raise an InvalidCLICommand when executed to provide the usage
+      # in the exception message
+      class NullRunner
+        def initialize(args)
+        end
+
+        def run
+          fail InvalidCLICommand.new("Usage: #{EXECUTABLE_NAME} <command> [options] [arguments...]\n\n"\
+                                     "Available commands are: #{GitLab::Monitor::CLI.commands.keys.join(', ')}")
+        end
+      end
+
+      # Git runner.
+      #
+      # Takes something that behaves like ARGV with optparse included as an argument.
+      #
+      # It will take 2 positional arguments once parsed, the first one for the repository location,
+      # the optional second one is an IO like object to write to
+      class GIT
+        COMMAND_NAME = "git".freeze
+
+        attr_reader :source, :target, :labels
+
+        def initialize(args)
+          @options = options(args)
+          args = @options.parse!
+          @source = args.shift
+          @target = args.shift || STDOUT
+          @labels ||= {}
+        end
+
+        def run
+          validate!
+
+          ::GitLab::Monitor::GitProber.new(labels: labels, source: source)
+                                      .probe_pull
+                                      .probe_push
+                                      .write_to(@target)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options] repository_path [target_file]"
+            opts.on("-l", "--labels=key=value,key2=value2", "Labels to append to the metrics") do |val|
+              @labels = val.split(",").map { |value| value.split("=").tap { |aa| aa[0] = aa[0].to_sym } }.to_h
+            end
+            opts
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def validate!
+          fail InvalidCLICommand.new(help) if @source.nil?
+          fail InvalidCLICommand.new("Can't find repository #{@source}\n\n#{help}") unless File.directory? @source
+        end
+      end
+
+      # Database tuple stats runner.
+      #
+      # It will take a database connection string and print results to STDOUT
+      class DatabaseTupleStats
+        COMMAND_NAME = "db-tuple-stats".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+
+          @target = args.shift || STDOUT
+          @target = File.open(@target, "a") if @target.is_a?(String)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
+              @db_connection_string = val
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          validate!
+
+          ::GitLab::Monitor::Database::TuplesProber.new(connection_string: @db_connection_string)
+                                                   .probe_db
+                                                   .write_to(@target)
+        end
+
+        private
+
+        def validate!
+          fail InvalidCLICommand.new(help) unless @db_connection_string
+        end
+      end
+
+      # Database row counts query runner.
+      #
+      # This will take the database connection and print the result to STDOUT
+      class DatabaseRowCounts
+        COMMAND_NAME = "row-counts".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+
+          @target = args.shift || STDOUT
+          @target = File.open(@target, "a") if @target.is_a?(String)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
+              @db_connection_string = val
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          validate!
+
+          ::GitLab::Monitor::Database::RowCountProber.new(connection_string: @db_connection_string)
+                                                     .probe_db
+                                                     .write_to(@target)
+        end
+
+        private
+
+        def validate!
+          fail InvalidCLICommand.new(help) unless @db_connection_string
+        end
+      end
+
+      # Run a web server that exposes the metrics specified in a config file
+      class Server
+        COMMAND_NAME = "web".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("-c config.yml", "Monitoring config") do |val|
+              @config_file = val
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          validate!
+
+          config = Utils.deep_symbolize_hash_keys(YAML.load_file(@config_file))
+
+          WebExporter.setup(config)
+          WebExporter.run!
+        end
+
+        private
+
+        def validate!
+          fail InvalidCLICommand.new(help) unless @config_file
+        end
+      end
+
+      # Process runner
+      #
+      # Takes a pid and name for metrics
+      class Process
+        COMMAND_NAME = "process".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+
+          @target = args.shift || STDOUT
+          @target = File.open(@target, "a") if @target.is_a?(String)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("--pid=123", "Process ID") do |val|
+              @pid = val
+            end
+            opts.on("--pattern=unicorn", "Process command pattern") do |val|
+              @pattern = val
+            end
+            opts.on("--name=NAME", "Process name to be used in metrics") do |val|
+              @name = val
+            end
+            opts.on("--quantiles", "Return quantiles instead of exact metrics") do
+              @quantiles = true
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          ::GitLab::Monitor::ProcessProber.new(pid_or_pattern: @pid || @pattern, name: @name, quantiles: @quantiles)
+                                          .probe_stat
+                                          .probe_count
+                                          .probe_smaps
+                                          .write_to(@target)
+        end
+      end
+
+      # Sidekiq runner.
+      #
+      # It will take a Redis connection URL and print results to STDOUT
+      class SidekiqRunner
+        COMMAND_NAME = "sidekiq".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+
+          @target = args.shift || STDOUT
+          @target = File.open(@target, "a") if @target.is_a?(String)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("--redis-url=\"redis://localhost:6379\"", "Redis URL") do |val|
+              @redis_url = val
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          validate!
+
+          ::GitLab::Monitor::SidekiqProber.new(redis_url: @redis_url)
+                                          .probe_queues
+                                          .probe_jobs
+                                          .probe_workers
+                                          .probe_retries
+                                          .write_to(@target)
+        end
+
+        private
+
+        def validate!
+          fail InvalidCLICommand.new(help) unless @redis_url
+        end
+      end
+
+      # Process runner
+      #
+      # Takes a pid and name for metrics
+      class GitProcess
+        COMMAND_NAME = "git-process".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+
+          @target = args.shift || STDOUT
+          @target = File.open(@target, "a") if @target.is_a?(String)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("--quantiles", "Return quantiles instead of exact metrics") do
+              @quantiles = true
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          ::GitLab::Monitor::GitProcessProber.new(quantiles: @quantiles)
+                                             .probe_git
+                                             .write_to(@target)
+        end
+      end
+
+      def self.commands
+        [
+          GIT,
+          DatabaseTupleStats,
+          DatabaseRowCounts,
+          Process,
+          GitProcess,
+          SidekiqRunner,
+          Server
+        ].each_with_object({}) do |command_class, commands|
+          commands[command_class::COMMAND_NAME] = command_class
+          commands
+        end
+      end
+    end
+  end
+end