gitlab-exporter 11.2.0 → 11.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d94bb457b87a9b44d9147c8bb0f504d43ca4d6516d0995b5f02fca2f2f19f71
-  data.tar.gz: 00bf2b35ff8759f55485026ff21c72adc472ad750001450084bbfee5f4e5f59d
+  metadata.gz: f427ed5f53c6b5beb7cf337bc7e8f3061729b1c485b43a2fef2825db07e10c58
+  data.tar.gz: b2d5c4398ead9a7051700f962867c677378a5b339510e8a79d8ed94a49f15bb3
 SHA512:
-  metadata.gz: 19769e5db26bf3be5a5dfe4109d4e7e36830d7e2bcc9a471398936cf5a83eb99a7b63082cacf826219f6e1d6ab229e6b97504670b402d74326e7e984ed9034c3
-  data.tar.gz: c65749449b07cc73b777c79d93091e6720392663574b627b48f0dc56ac44975903402cddaa9c081e2b064fe17252e0ecb35260b354e8dad18c5f9fbd76b96ffb
+  metadata.gz: 7c20308b95ec429aeb9dc166872105f1847d6cfe2dfc0a4a4f72c1f1c5c2659e8349e722fb7ae9b3405cd58912926804d29df9ef02fa5e9923e14443f1839dd1
+  data.tar.gz: d8c6811f60d8ecce68919500f546cad85ae64fb95a822379ab23f6c0ad41fb8fe2a857e03ca4e2986026afbc0179351939de64d6d09c5684d72330db49bd0f1a

data/.ruby-version

@@ -1 +1 @@
-2.7.2
+2.7.4

data/Gemfile.lock

@@ -1,14 +1,15 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (11.2.0)
+    gitlab-exporter (11.11.0)
       connection_pool (= 2.2.5)
+      faraday (~> 1.8.0)
       pg (= 1.2.3)
-      puma (= 5.3.2)
+      puma (= 5.6.2)
       quantile (= 0.2.1)
-      redis (= 4.1.4)
+      redis (= 4.4.0)
       redis-namespace (= 1.6.0)
-      sidekiq (= 5.2.9)
+      sidekiq (= 6.4.0)
       sinatra (~> 2.1.0)
 
 GEM
@@ -17,21 +18,41 @@ GEM
     ast (2.4.1)
     connection_pool (2.2.5)
     diff-lcs (1.3)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    multipart-post (2.1.1)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    nio4r (2.5.7)
+    nio4r (2.5.8)
     parallel (1.20.1)
     parser (3.0.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    puma (5.3.2)
+    puma (5.6.2)
       nio4r (~> 2.0)
     quantile (0.2.1)
     rack (2.2.3)
     rack-protection (2.1.0)
       rack
     rainbow (3.0.0)
-    redis (4.1.4)
+    redis (4.4.0)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
     regexp_parser (2.0.3)
@@ -61,12 +82,11 @@ GEM
     rubocop-ast (1.4.0)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
-    ruby2_keywords (0.0.4)
-    sidekiq (5.2.9)
-      connection_pool (~> 2.2, >= 2.2.2)
+    ruby2_keywords (0.0.5)
+    sidekiq (6.4.0)
+      connection_pool (>= 2.2.2)
       rack (~> 2.0)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 4.2)
+      redis (>= 4.2.0)
     sinatra (2.1.0)
       mustermann (~> 1.0)
       rack (~> 2.2)
@@ -87,4 +107,4 @@ DEPENDENCIES
   webrick (~> 1.7)
 
 BUNDLED WITH
-   2.2.22
+   2.2.29

data/README.md

@@ -59,6 +59,8 @@ metrics.
     * `sidekiq_schedule_set_backlog_count`
     * `sidekiq_retry_set_processing_delay_seconds`
     * `sidekiq_retry_set_backlog_count`
+1. [Elasticsearch](lib/gitlab_exporter/elasticsearch.rb)
+  * [Migrations](https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations) -- `elasticsearch_migrations_state`
 
 ### Setup with GitLab Development Kit
 

data/config/gitlab-exporter.yml.example

@@ -99,6 +99,22 @@ probes:
     opts:
       quantiles: false
 
+
+  # Uncomment below to add GitLab specific Elasticsearch metrics.
+  #
+  # To support multiple authorization types, opts are passed to a Faraday connection object.
+  # See: https://www.rubydoc.info/github/lostisland/faraday/Faraday%2FConnection:initialize
+  #
+  # elasticsearch: &elasticsearch
+  #   methods:
+  #     - probe_migrations
+  #
+  #   opts:
+  #     - url: http://localhost:9200
+  #       options:
+  #         headers:
+  #           Authorization: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="
+
   metrics:
     multiple: true
     git_process:

data/gitlab-exporter.gemspec

@@ -21,12 +21,13 @@ Gem::Specification.new do |s|
   s.license       = "MIT"
 
   s.add_runtime_dependency "connection_pool", "2.2.5"
+  s.add_runtime_dependency "faraday", "~> 1.8.0"
   s.add_runtime_dependency "pg", "1.2.3"
-  s.add_runtime_dependency "puma", "5.3.2"
+  s.add_runtime_dependency "puma", "5.6.2"
   s.add_runtime_dependency "quantile", "0.2.1"
-  s.add_runtime_dependency "redis", "4.1.4"
+  s.add_runtime_dependency "redis", "4.4.0"
   s.add_runtime_dependency "redis-namespace", "1.6.0"
-  s.add_runtime_dependency "sidekiq", "5.2.9"
+  s.add_runtime_dependency "sidekiq", "6.4.0"
   s.add_runtime_dependency "sinatra", "~> 2.1.0"
 
   s.add_development_dependency "rspec", "~> 3.7.0"

data/lib/gitlab_exporter/elasticsearch.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+
+module GitLab
+  module Exporter
+    # Exports GitLab specific Elasticsearch metrics.
+    #
+    # For generic operational metrics, see elasticsearch_exporter.
+    # https://github.com/prometheus-community/elasticsearch_exporter
+    class ElasticsearchProber
+      MIGRATION_STATE_MAP = {
+        unknown: -9,
+        # TODO: failed: -1
+        pending: 0,
+        running: 1,
+        halted: 2,
+        completed: 3
+      }.freeze
+
+      def initialize(metrics: PrometheusMetrics.new, logger: nil, **opts)
+        @metrics = metrics
+        @logger  = logger
+        @opts    = opts
+      end
+
+      # Probes the state of Advanced Search Migrations
+      # https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations
+      def probe_migrations
+        elastic_probe do |conn|
+          resp = conn.get "/gitlab-*-migrations/_search"
+          return unless resp.status == 200
+
+          JSON.parse(resp.body).dig("hits", "hits").each do |hit|
+            @metrics.add(
+              "elasticsearch_migrations_info", 1, # 1 is a noop.
+              state: inferred_migration_state(hit.fetch("_source")),
+              name: hit.fetch("_id")
+            )
+          end
+        end
+      rescue StandardError => e
+        @logger&.error "ElasticsearchProper encountered an error: #{e}"
+      end
+
+      private
+
+      def elastic_probe
+        yield Faraday.new(@opts.fetch(:url), @opts.fetch(:options, {}))
+      end
+
+      def inferred_migration_state(migration)
+        return :pending if migration["started_at"] == ""
+
+        if migration["started_at"] != "" && migration["completed_at"] == "" && !migration.dig("state", "halted")
+          return :running
+        end
+
+        return :completed if migration["completed"]
+        return :halted if migration.dig("state", "halted")
+
+        @logger&.error("Elasticsearch probe doesn't know the state of a migration")
+        :unknown
+      end
+    end
+  end
+end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "11.2.0".freeze
+    VERSION = "11.11.0".freeze
   end
 end

data/lib/gitlab_exporter.rb

@@ -1,19 +1,20 @@
 module GitLab
   # GitLab Monitoring
   module Exporter
-    autoload :CLI,               "gitlab_exporter/cli"
-    autoload :TimeTracker,       "gitlab_exporter/util"
-    autoload :Utils,             "gitlab_exporter/util"
-    autoload :PrometheusMetrics, "gitlab_exporter/prometheus"
-    autoload :Utils,             "gitlab_exporter/util"
-    autoload :Git,               "gitlab_exporter/git"
-    autoload :GitProber,         "gitlab_exporter/git"
-    autoload :GitProcessProber,  "gitlab_exporter/git"
-    autoload :Database,          "gitlab_exporter/database"
-    autoload :ProcessProber,     "gitlab_exporter/process"
-    autoload :WebExporter,       "gitlab_exporter/web_exporter"
-    autoload :Prober,            "gitlab_exporter/prober"
-    autoload :SidekiqProber,     "gitlab_exporter/sidekiq"
-    autoload :RubyProber,        "gitlab_exporter/ruby"
+    autoload :CLI,                 "gitlab_exporter/cli"
+    autoload :TimeTracker,         "gitlab_exporter/util"
+    autoload :Utils,               "gitlab_exporter/util"
+    autoload :PrometheusMetrics,   "gitlab_exporter/prometheus"
+    autoload :Utils,               "gitlab_exporter/util"
+    autoload :Git,                 "gitlab_exporter/git"
+    autoload :GitProber,           "gitlab_exporter/git"
+    autoload :GitProcessProber,    "gitlab_exporter/git"
+    autoload :Database,            "gitlab_exporter/database"
+    autoload :ProcessProber,       "gitlab_exporter/process"
+    autoload :WebExporter,         "gitlab_exporter/web_exporter"
+    autoload :Prober,              "gitlab_exporter/prober"
+    autoload :SidekiqProber,       "gitlab_exporter/sidekiq"
+    autoload :RubyProber,          "gitlab_exporter/ruby"
+    autoload :ElasticsearchProber, "gitlab_exporter/elasticsearch"
   end
 end

data/spec/elasticsearch_spec.rb

@@ -0,0 +1,132 @@
+require "spec_helper"
+require "gitlab_exporter/elasticsearch"
+
+describe GitLab::Exporter::ElasticsearchProber do
+  subject(:prober) { GitLab::Exporter::Prober.new(**options) }
+  let(:options) do
+    {
+      elasticsearch: {
+        class_name: described_class.to_s,
+        methods: %i[probe_migrations],
+        opts: {
+          url: "http://elasticsearch"
+        }
+      }
+    }
+  end
+
+  let(:connection) { instance_double(Faraday::Connection) }
+
+  describe "probe_migrations" do
+    let(:response) { double(:response, body: results, status: 200) }
+    let(:metric_name) { "elasticsearch_migrations_info" }
+
+    let(:results) do
+      <<-RESULTS
+      {
+        "took": 5,
+        "timed_out": false,
+        "_shards": {
+          "total": 1,
+          "successful": 1,
+          "skipped": 0,
+          "failed": 0
+        },
+        "hits": {
+          "total": {
+            "value": 1,
+            "relation": "eq"
+          },
+          "max_score": 1,
+          "hits": [#{document.to_json}]
+        }
+      }
+      RESULTS
+    end
+
+    let(:document) do
+      { "_id" => "123", "_source" => migration }
+    end
+
+    let(:migration) do
+      {
+        "started_at" => "",
+        "completed_at" => "",
+        "completed" => false,
+        "state" => {}
+      }.tap do |m|
+        case migration_state
+        when "running"
+          m["started_at"] = "2021-09-23T11:47:48,890Z"
+        when "completed"
+          m["started_at"] = "2021-09-23T11:47:48,890Z"
+          m["completed_at"] = "2021-09-24T11:47:48,890Z"
+          m["completed"] = true
+        when "halted"
+          m["started_at"] = "2021-09-23T11:47:48,890Z"
+          m["state"]["halted"] = true
+        end
+      end
+    end
+
+    let(:output) do
+      prober.probe_all
+      StringIO.new.tap do |s|
+        prober.write_to(s)
+      end
+    end
+
+    before do
+      allow_any_instance_of(described_class).to receive(:elastic_probe).and_yield(connection)
+    end
+
+    describe "when an Exception is raised" do
+      before do
+        allow(connection).to receive(:get).with("/gitlab-*-migrations/_search").and_raise("boom")
+      end
+
+      it "does NOT raise an error" do
+        expect { output }.to_not raise_error
+      end
+
+      it "does NOT export any metrics" do
+        expect(output.string).not_to match(/#{metric_name}/)
+      end
+    end
+
+    describe "when elasticsearch is reachable" do
+      let(:expected_output) { /#{metric_name}{state="#{migration_state}",name="123"} 1 \d+/ }
+      before do
+        allow(connection).to receive(:get).with("/gitlab-*-migrations/_search").and_return response
+      end
+
+      context "when there is a pending migration" do
+        let(:migration_state) { "pending" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+
+      context "when there is a running migration" do
+        let(:migration_state) { "running" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+
+      context "when there is a halted migration" do
+        let(:migration_state) { "halted" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+
+      context "when there is a completed migration" do
+        let(:migration_state) { "completed" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 11.2.0
+  version: 11.11.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -24,6 +24,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.2.5
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.3.2
+        version: 5.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.3.2
+        version: 5.6.2
 - !ruby/object:Gem::Dependency
   name: quantile
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +86,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.1.4
+        version: 4.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.1.4
+        version: 4.4.0
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +114,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.9
+        version: 6.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.9
+        version: 6.4.0
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
@@ -181,6 +195,7 @@ files:
 - lib/gitlab_exporter/database/remote_mirrors.rb
 - lib/gitlab_exporter/database/row_count.rb
 - lib/gitlab_exporter/database/tuple_stats.rb
+- lib/gitlab_exporter/elasticsearch.rb
 - lib/gitlab_exporter/git.rb
 - lib/gitlab_exporter/memstats.rb
 - lib/gitlab_exporter/memstats/mapping.rb
@@ -196,6 +211,7 @@ files:
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
+- spec/elasticsearch_spec.rb
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
@@ -223,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
@@ -232,6 +248,7 @@ test_files:
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
+- spec/elasticsearch_spec.rb
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb