gitlab-exporter 17.1.0 → 17.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df145039780458bab34b436ca149dce37c4c4f4a08e1707b606f70dc4aa021a4
-  data.tar.gz: 272f4e78ffb5901954f086e58a158087328d0478deb18114fb748504e067796b
+  metadata.gz: 9f67afb938a5a207bd67f7402559c6bf2c62e8c48e2d24660c052361727e862a
+  data.tar.gz: ec77e34ad3640e64d368a00a844d91e651d64d1c300a8f8a82de3c4c0567fb8b
 SHA512:
-  metadata.gz: a223ec3d899fcc85eeb391e100cce3942aeb050264cc03c6877ad6387f09234461408f45f1a8a82128c657b240f4ce2768c35a1d58857d4c3fb212f42090f28c
-  data.tar.gz: 992f12485329239412a793b67c556c7226535765d3ecfea83271e055c679c75d7d1d0af8b0be71e83408a0aacc77ceb0e3d9079281725d26b04f86955e01baf7
+  metadata.gz: 17a4dd08119dc1f47eb6f6867b466e91090ff3b673ed3d703792e11764dff5a8686c84ad22cc19168bdd9ec66759260956286138bef79bbcece8fc3928c476c9
+  data.tar.gz: bad62ae9548056c970ce2b4c0fb84aa3d60879e397f17e7e97fa0c1a98116174ae7ea6b6d684972b01d1d4f4301fa1c892ec733850ef1fccde08fb0367d15f0f

data/CONTRIBUTING.md

@@ -61,7 +61,7 @@ abbreviation.
 If you have read this guide and want to know how the GitLab [core team]
 operates please see [the GitLab contributing process](PROCESS.md).
 
-- [GitLab Inc engineers should refer to the engineering workflow document](https://about.gitlab.com/handbook/engineering/workflow/)
+- [GitLab Inc engineers should refer to the engineering workflow document](https://handbook.gitlab.com/handbook/engineering/workflow/)
 
 ## Security vulnerability disclosure
 
@@ -266,7 +266,7 @@ on those issues. Please select someone with relevant experience from the
 [GitLab team][team]. If there is nobody mentioned with that expertise look in
 the commit history for the affected files to find someone.
 
-[described in our handbook]: https://about.gitlab.com/handbook/engineering/issues/issue-triage-policies/
+[described in our handbook]: https://handbook.gitlab.com/handbook/product-development/how-we-work/issue-triage/
 [issue bash events]: https://gitlab.com/gitlab-org/gitlab-ce/issues/17815
 
 ### Feature proposals
@@ -512,7 +512,7 @@ When having your code reviewed and when reviewing merge requests please take the
    making and testing future changes
 1. Changes do not adversely degrade performance.
    - Avoid repeated polling of endpoints that require a significant amount of overhead
-   - Check for N+1 queries via the SQL log or [`QueryRecorder`](https://docs.gitlab.com/ce/development/merge_request_performance_guidelines.html)
+   - Check for N+1 queries via the SQL log or [`QueryRecorder`](https://docs.gitlab.com/development/database/query_recorder/)
    - Avoid repeated access of filesystem
 1. If you need polling to support real-time features, please use
    [polling with ETag caching][polling-etag].
@@ -641,10 +641,9 @@ available at [http://contributor-covenant.org/version/1/1/0/](http://contributor
 [js-styleguide]: doc/development/fe_guide/style_guide_js.md "JavaScript styleguide"
 [scss-styleguide]: doc/development/fe_guide/style_guide_scss.md "SCSS styleguide"
 [newlines-styleguide]: doc/development/newlines_styleguide.md "Newlines styleguide"
-[UX Guide for GitLab]: http://docs.gitlab.com/ce/development/ux_guide/
+[UX Guide for GitLab]: https://design.gitlab.com/
 [license-finder-doc]: doc/development/licensing.md
-[GitLab Inc engineering workflow]: https://about.gitlab.com/handbook/engineering/workflow/#labelling-issues
-[polling-etag]: https://docs.gitlab.com/ce/development/polling.html
+[polling-etag]: https://docs.gitlab.com/development/polling/
 [testing]: doc/development/testing.md
 
 [^1]: Please note that specs other than JavaScript specs are considered backend

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (17.1.0)
+    gitlab-exporter (17.2.0)
       base64 (= 0.3.0)
       connection_pool (= 2.5.5)
       deep_merge (~> 1.2.2)
-      faraday (= 2.14.4)
+      faraday (= 2.15.1)
       pg (= 1.6.3)
       quantile (= 0.2.1)
       redis (= 4.8.1)
@@ -21,8 +21,8 @@ GEM
     connection_pool (2.5.5)
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
-    faraday (2.14.4)
-      faraday-net_http (>= 2.0, < 3.5)
+    faraday (2.15.1)
+      faraday-net_http (>= 2.0, < 3.6)
       json
       logger
     faraday-net_http (3.0.2)
@@ -43,7 +43,7 @@ GEM
     redis (4.8.1)
     redis-namespace (1.11.0)
       redis (>= 4)
-    regexp_parser (2.11.0)
+    regexp_parser (2.13.1)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -57,14 +57,14 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.91.0)
+    rubocop (1.92.2)
       json (>= 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
       parallel (>= 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.9.3, < 3.0)
+      regexp_parser (>= 2.11.3, < 3.0)
       rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
@@ -91,4 +91,4 @@ DEPENDENCIES
   rubocop (~> 1.0)
 
 BUNDLED WITH
-  4.0.18
+  4.0.19

data/README.md

@@ -38,7 +38,19 @@ metrics.
       Usage-data counters are excluded. Enable the `geo_node_status` probe in the `database`
       endpoint configuration.
     * [Zoekt](lib/gitlab_exporter/database/zoekt.rb) --
-      `search_zoekt_task_processing_queue_size`
+      `search_zoekt_task_processing_queue_size`,
+      `search_zoekt_repositories_schema_version_count`,
+      `search_zoekt_nodes_status`,
+      `search_zoekt_node_unclaimed_storage_bytes`,
+      `search_zoekt_node_storage_percent_used`,
+      `search_zoekt_node_enabled_namespaces`,
+      `search_zoekt_node_tasks`,
+      `search_zoekt_repositories_states_total`,
+      `search_zoekt_indices_states`,
+      `search_zoekt_indices_watermark_levels`,
+      `search_zoekt_indices_reserved_storage_bytes`,
+      `search_zoekt_indices_used_storage_bytes`,
+      `search_zoekt_indices_with_stale_used_storage_bytes`
 1. Git
     * [git pull/push timings](lib/gitlab_exporter/git.rb) --
       `git_pull_time_milliseconds`, `git_push_time_milliseconds`
@@ -70,7 +82,7 @@ metrics.
     * `sidekiq_retry_set_processing_delay_seconds`
     * `sidekiq_retry_set_backlog_count`
 1. [Elasticsearch](lib/gitlab_exporter/elasticsearch.rb)
-  * [Migrations](https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations) -- `elasticsearch_migrations_state`
+  * [Migrations](https://docs.gitlab.com/integration/advanced_search/elasticsearch/#advanced-search-migrations) -- `elasticsearch_migrations_info`
 
 ### Setup with GitLab Development Kit
 
@@ -105,6 +117,34 @@ Once running, you can point your browser or curl to the following URLs:
 * http://localhost:9168/sidekiq
 * http://localhost:9168/metrics (to get all of the above combined)
 
+#### Health checks
+
+The web exporter also serves two static routes for health checks, such as
+Kubernetes liveness and readiness probes:
+
+* `/-/liveness`
+* `/-/readiness`
+
+Both return `200 OK` with a plain-text `OK` body. They run no probes and
+make no connections to PostgreSQL, Redis, or Sidekiq, so a backend outage
+shows up in the metrics instead of restarting the exporter. The server
+accepts requests only after all probe routes are mounted, so readiness
+currently behaves the same as liveness.
+
+The routes are served by the same listener as the metrics, so set
+`scheme: HTTPS` on the probes when `tls_enabled` is set. For example:
+
+```yaml
+livenessProbe:
+  httpGet:
+    path: /-/liveness
+    port: 9168
+readinessProbe:
+  httpGet:
+    path: /-/readiness
+    port: 9168
+```
+
 If it is undesirable to have secrets (e.g. Redis password or PostgreSQL credentials) in the config file,
 then you can specify an external command (with the `--extra-config-command <command>` flag) that outputs
 such credentials in a YAML form (same structure as the config file) and it will be merged with the file configuration.

data/config/gitlab-exporter.yml.example

@@ -77,9 +77,9 @@ probes:
         <<: *db_common_opts
         project_ids:
         - 1
-      geo_node_status:
-        class_name: Database::GeoNodeStatusProber
-        <<: *db_common
+    geo_node_status:
+      class_name: Database::GeoNodeStatusProber
+      <<: *db_common
     zoekt:
       class_name: Database::ZoektProber
       <<: *db_common

data/gitlab-exporter.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "base64", "0.3.0"
   s.add_runtime_dependency "connection_pool", "2.5.5"
   s.add_runtime_dependency "deep_merge", "~> 1.2.2"
-  s.add_runtime_dependency "faraday", "2.14.4"
+  s.add_runtime_dependency "faraday", "2.15.1"
   s.add_runtime_dependency "pg", "1.6.3"
   s.add_runtime_dependency "quantile", "0.2.1"
   s.add_runtime_dependency "redis", "4.8.1"

data/lib/gitlab_exporter/cli.rb

@@ -176,8 +176,9 @@ module GitLab
             opts.on("-c config.yml", "Monitoring config") do |val|
               @config_file = val
             end
-            opts.on("--extra-config-command command", "Shell command that returns YAML config to be merged
-                    with the config file") do |val|
+            opts.on("--extra-config-command command",
+                    "Shell command that returns YAML config to be merged",
+                    "with the config file") do |val|
               @extra_config_command = val
             end
           end

data/lib/gitlab_exporter/elasticsearch.rb

@@ -26,7 +26,7 @@ module GitLab
       end
 
       # Probes the state of Advanced Search Migrations
-      # https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations
+      # https://docs.gitlab.com/integration/advanced_search/elasticsearch/#advanced-search-migrations
       def probe_migrations
         elastic_probe do |conn|
           resp = conn.get "/gitlab-*-migrations/_search"

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "17.1.0".freeze
+    VERSION = "17.2.0".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -14,6 +14,11 @@ module GitLab
       DEFAULT_LISTEN_PORT = 9168
       DEFAULT_MEMORY_THRESHOLD = 1024
 
+      # Static routes for Kubernetes liveness and readiness probes. They run no
+      # probers, so a slow or failing backend never restarts the pod or removes it
+      # from Service endpoints. The `/-/` prefix keeps them apart from probe routes.
+      HEALTH_ROUTES = %w[/-/liveness /-/readiness].freeze
+
       # Security response headers previously added by Sinatra's Rack::Protection
       # middleware, which is gone now that we serve directly via WEBrick. Kept for
       # parity so scanners and consumers see the same headers as before.
@@ -34,6 +39,7 @@ module GitLab
 
           @server = build_server(server_config)
           mount_probes(config[:probes])
+          mount_health_routes
 
           # Defrag heap after everything is loaded into memory.
           GC.compact
@@ -90,6 +96,29 @@ module GitLab
           end
         end
 
+        # Mounted after the probes so these routes win over a probe that happens to
+        # share a path. The server only accepts requests once `run!` starts it,
+        # after `setup` has returned, so readiness currently matches liveness.
+        def mount_health_routes
+          HEALTH_ROUTES.each do |path|
+            @server.mount_proc(path) do |_request, response|
+              handle_health(response)
+            end
+          end
+        end
+
+        def handle_health(response)
+          write_response(response, "text/plain; charset=utf-8", "OK")
+        end
+
+        def write_response(response, content_type, body)
+          SECURITY_HEADERS.each do |header, value|
+            response[header] = value
+          end
+          response.content_type = content_type
+          response.body = body
+        end
+
         def handle_probe(response, opts)
           # A memory killer that kills the process if we exceeded a certain threshold.
           check_memory_threshold!
@@ -100,11 +129,7 @@ module GitLab
           buffer = StringIO.new
           prober.write_to(buffer)
 
-          SECURITY_HEADERS.each do |header, value|
-            response[header] = value
-          end
-          response.content_type = "text/plain; version=0.0.4; charset=utf-8"
-          response.body = buffer.string
+          write_response(response, "text/plain; version=0.0.4; charset=utf-8", buffer.string)
         ensure
           # Performs a major GC after each request. We found that this helps to free up
           # several MB of memory in conjunction with stricter malloc config.

data/spec/cli_spec.rb

@@ -32,6 +32,18 @@ context "With valid pair of repositories" do
 end
 
 describe GitLab::Exporter::CLI::Server do
+  describe "#help" do
+    it "aligns the --extra-config-command description with the other options" do
+      help = described_class.new([].extend(OptionParser::Arguable)).help
+
+      expect(help).to include(
+        "        --extra-config-command command\n" \
+        "                                     Shell command that returns YAML config to be merged\n" \
+        "                                     with the config file\n"
+      )
+    end
+  end
+
   context "extra-config-command is passed" do
     let(:argv) do
       ["-c", "spec/fixtures/config.yml", "--extra-config-command", extra_config_command]

data/spec/config_spec.rb

@@ -0,0 +1,36 @@
+require "spec_helper"
+require "yaml"
+
+describe "config/gitlab-exporter.yml.example" do
+  let(:config_path) { File.expand_path("../config/gitlab-exporter.yml.example", __dir__) }
+  let(:config) { YAML.safe_load_file(config_path, aliases: true) }
+
+  it "parses as valid YAML with aliases" do
+    expect { config }.not_to raise_error
+    expect(config).to be_a(Hash)
+  end
+
+  describe "database probes" do
+    let(:database_probes) { config.dig("probes", "database") }
+
+    it "configures geo_node_status directly under probes.database" do
+      expect(database_probes).to include("geo_node_status")
+      expect(database_probes["geo_node_status"]).to include(
+        "class_name" => "Database::GeoNodeStatusProber",
+        "methods" => ["probe_db"]
+      )
+    end
+
+    it "keeps remote_mirrors and geo_node_status as sibling probes without nested leakage" do
+      expect(database_probes["remote_mirrors"]).not_to have_key("geo_node_status")
+      expect(database_probes.keys).to include(
+        "ci_builds",
+        "tuple_stats",
+        "rows_count",
+        "remote_mirrors",
+        "geo_node_status",
+        "zoekt"
+      )
+    end
+  end
+end

data/spec/web_exporter_spec.rb

@@ -2,6 +2,8 @@ require "spec_helper"
 require "gitlab_exporter/web_exporter"
 require "net/http"
 require "timeout"
+require "openssl"
+require "tmpdir"
 
 describe GitLab::Exporter::WebExporter do
   # Each test builds a fresh server via `.setup`; drop the reference afterwards
@@ -29,9 +31,22 @@ describe GitLab::Exporter::WebExporter do
       server = instance_double(WEBrick::HTTPServer)
       allow(WEBrick::HTTPServer).to receive(:new).and_return(server)
 
-      expect(server).to receive(:mount_proc).with("/ruby")
+      allow(server).to receive(:mount_proc)
 
       described_class.setup(server: {}, probes: { ruby: { methods: %w[probe_gc], opts: {} } })
+
+      expect(server).to have_received(:mount_proc).with("/ruby")
+    end
+
+    it "mounts the health routes after the probes" do
+      server = instance_double(WEBrick::HTTPServer)
+      allow(WEBrick::HTTPServer).to receive(:new).and_return(server)
+      mounted = []
+      allow(server).to receive(:mount_proc) { |path| mounted << path }
+
+      described_class.setup(server: {}, probes: { ruby: { methods: %w[probe_gc], opts: {} } })
+
+      expect(mounted).to eq(%w[/ruby /-/liveness /-/readiness])
     end
 
     it "raises when TLS is enabled but no certificate is configured" do
@@ -45,7 +60,7 @@ describe GitLab::Exporter::WebExporter do
       captured = nil
       allow(WEBrick::HTTPServer).to receive(:new) do |**options|
         captured = options
-        instance_double(WEBrick::HTTPServer)
+        instance_double(WEBrick::HTTPServer, mount_proc: nil)
       end
 
       described_class.setup(config.merge(probes: {}))
@@ -123,5 +138,81 @@ describe GitLab::Exporter::WebExporter do
 
       expect(response.code).to eq("404")
     end
+
+    %w[/-/liveness /-/readiness].each do |path|
+      describe path do
+        it "returns a static 200 without running any probers" do
+          expect(GitLab::Exporter::Prober).not_to receive(:new)
+
+          response = Net::HTTP.get_response(URI("#{base_uri}#{path}"))
+
+          expect(response.code).to eq("200")
+          expect(response.content_type).to eq("text/plain")
+          expect(response.body).to eq("OK")
+          expect(response["X-Content-Type-Options"]).to eq("nosniff")
+        end
+
+        it "does not check the memory threshold" do
+          expect(described_class).not_to receive(:check_memory_threshold!)
+
+          Net::HTTP.get_response(URI("#{base_uri}#{path}"))
+        end
+
+        it "answers HEAD requests without a body" do
+          uri = URI("#{base_uri}#{path}")
+          response = Net::HTTP.start(uri.host, uri.port) { |http| http.head(uri.path) }
+
+          expect(response.code).to eq("200")
+          expect(response.body).to be_nil
+        end
+      end
+    end
+
+    context "with TLS enabled" do
+      let(:tls_dir) { Dir.mktmpdir }
+      let(:config) do
+        key = OpenSSL::PKey::RSA.new(2048)
+        cert = self_signed_cert(key)
+        File.write(File.join(tls_dir, "tls.key"), key.to_pem)
+        File.write(File.join(tls_dir, "tls.crt"), cert.to_pem)
+
+        {
+          server: {
+            listen_address: "127.0.0.1",
+            listen_port: 0,
+            tls_enabled: true,
+            tls_cert_path: File.join(tls_dir, "tls.crt"),
+            tls_key_path: File.join(tls_dir, "tls.key")
+          },
+          probes: {}
+        }
+      end
+
+      after do
+        FileUtils.remove_entry(tls_dir)
+      end
+
+      it "serves the health routes over HTTPS" do
+        port = server.listeners.first.addr[1]
+        response = Net::HTTP.start("127.0.0.1", port, use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE) { |http|
+          http.get("/-/liveness")
+        }
+
+        expect(response.code).to eq("200")
+        expect(response.body).to eq("OK")
+      end
+
+      def self_signed_cert(key)
+        cert = OpenSSL::X509::Certificate.new
+        cert.version = 2
+        cert.serial = 1
+        cert.subject = cert.issuer = OpenSSL::X509::Name.parse("/CN=127.0.0.1")
+        cert.public_key = key.public_key
+        cert.not_before = Time.now
+        cert.not_after = Time.now + 3600
+        cert.sign(key, OpenSSL::Digest.new("SHA256"))
+        cert
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 17.1.0
+  version: 17.2.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.14.4
+        version: 2.15.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.14.4
+        version: 2.15.1
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -229,6 +229,7 @@ files:
 - lib/gitlab_exporter/version.rb
 - lib/gitlab_exporter/web_exporter.rb
 - spec/cli_spec.rb
+- spec/config_spec.rb
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
 - spec/database/geo_node_status_spec.rb
@@ -274,6 +275,7 @@ specification_version: 4
 summary: GitLab metrics exporter
 test_files:
 - spec/cli_spec.rb
+- spec/config_spec.rb
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
 - spec/database/geo_node_status_spec.rb