gitlab-monitor 4.0.0

data/lib/gitlab_monitor/memstats/mapping.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module GitLab
+  module Monitor
+    module MemStats
+      # Parses one entry in /proc/[pid]/smaps. For example:
+      #
+      # 00400000-00401000 r-xp 00000000 08:01 541055                             /opt/gitlab/embedded/bin/ruby
+      # Size:                  4 kB
+      # Rss:                   4 kB
+      # Pss:                   0 kB
+      # Shared_Clean:          4 kB
+      # Shared_Dirty:          0 kB
+      # Private_Clean:         0 kB
+      # Private_Dirty:         0 kB
+      # Referenced:            4 kB
+      # Anonymous:             0 kB
+      # AnonHugePages:         0 kB
+      # Shared_Hugetlb:        0 kB
+      # Private_Hugetlb:       0 kB
+      # Swap:                  0 kB
+      # SwapPss:               0 kB
+      # KernelPageSize:        4 kB
+      # MMUPageSize:           4 kB
+      # Locked:                0 kB
+      # VmFlags: rd ex mr mw me dw sd
+      class Mapping
+        FIELDS = %w(size rss shared_clean shared_dirty private_clean private_dirty swap pss).freeze
+
+        attr_reader :address_start
+        attr_reader :address_end
+        attr_reader :perms
+        attr_reader :offset
+        attr_reader :device_major
+        attr_reader :device_minor
+        attr_reader :inode
+        attr_reader :region
+
+        attr_accessor :size
+        attr_accessor :rss
+        attr_accessor :shared_clean
+        attr_accessor :shared_dirty
+        attr_accessor :private_dirty
+        attr_accessor :private_clean
+        attr_accessor :swap
+        attr_accessor :pss
+
+        def initialize(lines)
+          FIELDS.each do |field|
+            send("#{field}=", 0)
+          end
+
+          parse_first_line(lines.shift)
+
+          lines.each do |l|
+            parse_field_line(l)
+          end
+        end
+
+        def parse_first_line(line)
+          line.strip!
+
+          parts = line.split
+          @address_start, @address_end = parts[0].split("-")
+          @perms = parts[1]
+          @offset = parts[2]
+          @device_major, @device_minor = parts[3].split(":")
+          @inode = parts[4]
+          @region = parts[5] || "anonymous"
+        end
+
+        def parse_field_line(line)
+          line.strip!
+
+          parts = line.split
+
+          return unless parts
+
+          parts[0].downcase!
+          parts[0].sub!(":", "")
+          field = parts[0]
+
+          return unless respond_to? "#{field}="
+
+          value = Float(parts[1]).to_i
+          send("#{field}=", value)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/prober.rb

@@ -0,0 +1,40 @@
+module GitLab
+  module Monitor
+    # A class to combine multiple probers into one
+    class Prober
+      def initialize(prober_opts, metrics: PrometheusMetrics.new)
+        @prober_opts = prober_opts
+        @metrics     = metrics
+
+        resolve_prober_classes
+      end
+
+      def probe_all
+        @prober_opts.each do |_probe_name, params|
+          Utils.wrap_in_array(params[:opts]).each do |opts|
+            prober = params[:class].new(opts, metrics: @metrics)
+            params[:methods].each do |meth|
+              prober.send(meth)
+            end
+          end
+        end
+      end
+
+      def write_to(target)
+        target.write(@metrics.to_s)
+      end
+
+      private
+
+      def resolve_prober_classes
+        @prober_opts.each do |probe_name, params|
+          prober_class_name = params[:class_name] || Utils.camel_case_string("#{probe_name}_prober")
+
+          klass = prober_class_name.split("::").reduce(GitLab::Monitor) { |a, e| a.const_get(e) }
+
+          params[:class] = klass
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/process.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "memstats"
+
+module GitLab
+  module Monitor
+    # A helper class to extract memory info from /proc/<pid>/status
+    #
+
+    # A helper class to stats from /proc/<pid>/stat
+    #
+    # See: man 5 proc
+    #
+    # It takes a pid
+    class ProcessStats
+      def initialize(pid)
+        @pid = pid
+        @user_hertz = retrieve_user_hertz
+        @stats = populate_info
+      end
+
+      def valid?
+        !@stats.nil?
+      end
+
+      def cpu_time
+        (@stats[14].to_i + @stats[15].to_i) / @user_hertz
+      end
+
+      def start_time
+        @stats[22].to_i / @user_hertz
+      end
+
+      def vsize
+        # Virtual memory size in bytes.
+        @stats[23].to_i
+      end
+
+      def rss
+        # Resident Set Size: number of pages the process has in real memory.
+        @stats[24].to_i * 4096
+      end
+
+      private
+
+      def populate_info
+        # Pad the array by one element to make field numbers match the man page.
+        [""].concat(File.read("/proc/#{@pid}/stat").split(" "))
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def retrieve_user_hertz
+        Process.clock_getres(:TIMES_BASED_CLOCK_PROCESS_CPUTIME_ID, :hertz)
+      rescue Errno::EINVAL
+        100.0
+      end
+    end
+
+    # Probes a process for info then writes metrics to a target
+    class ProcessProber
+      def initialize(options, metrics: PrometheusMetrics.new)
+        @metrics = metrics
+        @name    = options[:name]
+        @pids    = if options[:pid_or_pattern] =~ /^\d+$/
+                     [options[:pid_or_pattern]]
+                   else
+                     Utils.pgrep(options[:pid_or_pattern])
+                   end
+        @use_quantiles = options.fetch(:quantiles, false)
+      end
+
+      def probe_stat
+        @pids.each do |pid|
+          stats = ProcessStats.new(pid)
+          next unless stats.valid?
+
+          labels = { name: @name.downcase }
+          labels[:pid] = pid unless @use_quantiles
+
+          @metrics.add("process_cpu_seconds_total", stats.cpu_time, @use_quantiles, **labels)
+          @metrics.add("process_resident_memory_bytes", stats.rss, @use_quantiles, **labels)
+          @metrics.add("process_virtual_memory_bytes", stats.vsize, @use_quantiles, **labels)
+          @metrics.add("process_start_time_seconds", stats.start_time, @use_quantiles, **labels)
+        end
+
+        self
+      end
+
+      def probe_count
+        @metrics.add("process_count", @pids.count, name: @name.downcase)
+
+        self
+      end
+
+      def probe_smaps
+        @pids.each do |pid|
+          stats = ::GitLab::Monitor::MemStats::Aggregator.new(pid)
+
+          next unless stats.valid?
+
+          labels = { name: @name.downcase }
+          labels[:pid] = pid unless @use_quantiles
+
+          ::GitLab::Monitor::MemStats::Mapping::FIELDS.each do |field|
+            value = stats.totals[field]
+
+            if value >= 0
+              @metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels)
+            end
+          end
+        end
+
+        self
+      end
+
+      def write_to(target)
+        target.write(@metrics.to_s)
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/prometheus.rb

@@ -0,0 +1,64 @@
+require "quantile"
+
+module GitLab
+  module Monitor
+    # Prometheus metrics container
+    #
+    # Provides a simple API to `add` metrics and then turn them `to_s` which will just
+    # dump all the metrics in prometheus format
+    #
+    # The add method also can take any arbitrary amount of labels in a `key: value` format.
+    class PrometheusMetrics
+      def initialize(include_timestamp: true)
+        @metrics = Hash.new { |h, k| h[k] = [] }
+        @quantiles = Hash.new { |h, k| h[k] = [] }
+        @include_timestamp = include_timestamp
+      end
+
+      def add(name, value, quantile = false, **labels)
+        if quantile
+          @quantiles[{ name: name, labels: labels }] << value
+        else
+          @metrics[name] << { value: value, labels: labels, timestamp: (Time.now.to_f * 1000).to_i }
+        end
+
+        self
+      end
+
+      def to_s
+        add_quantiles_to_metrics
+
+        buffer = ""
+        @metrics.each do |name, measurements|
+          measurements.each do |measurement|
+            buffer << name.to_s
+            labels = (measurement[:labels] || {}).map { |label, value| "#{label}=\"#{value}\"" }.join(",")
+            buffer << "{#{labels}}" unless labels.empty?
+            buffer << " #{measurement[:value]}"
+            buffer << " #{measurement[:timestamp]}" if @include_timestamp
+            buffer << "\n"
+          end
+        end
+        buffer
+      end
+
+      private
+
+      def add_quantiles_to_metrics
+        @quantiles.each do |data, measurements|
+          estimator = Quantile::Estimator.new
+
+          measurements.each do |value|
+            estimator.observe(value)
+          end
+
+          estimator.invariants.each do |invariant|
+            data[:labels][:quantile] = "#{(invariant.quantile * 100).to_i}th"
+
+            add(data[:name], estimator.query(invariant.quantile), **data[:labels])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/sidekiq.rb

@@ -0,0 +1,149 @@
+require "sidekiq/api"
+require "digest"
+
+module GitLab
+  module Monitor
+    # A prober for Sidekiq queues
+    #
+    # It takes the Redis URL Sidekiq is connected to
+    class SidekiqProber
+      QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
+      QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
+
+      def initialize(opts, metrics: PrometheusMetrics.new)
+        @opts    = opts
+        @metrics = metrics
+
+        Sidekiq.configure_client do |config|
+          config.redis = redis_options
+        end
+
+        ensure_queue_job_stats_script_loaded
+      end
+
+      def probe_queues
+        return self unless connected?
+
+        Sidekiq::Queue.all.each do |queue|
+          @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
+          @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
+          @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
+        end
+
+        self
+      end
+
+      def probe_jobs # rubocop:disable Metrics/MethodLength
+        return self unless connected?
+
+        job_stats = {}
+
+        Sidekiq::Queue.all.each do |queue|
+          begin
+            Sidekiq.redis do |conn|
+              stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
+              job_stats.merge!(stats.to_h)
+            end
+          rescue Redis::CommandError # Could happen if the script exceeded the maximum run time (5 seconds by default)
+            # FIXME: Should we call SCRIPT KILL?
+            return self
+          end
+        end
+
+        job_stats.each do |class_name, count|
+          @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
+        end
+
+        self
+      end
+
+      def probe_workers
+        return self unless connected?
+
+        worker_stats = Hash.new(0)
+
+        Sidekiq::Workers.new.map do |_pid, _tid, work|
+          job_klass = work["payload"]["class"]
+
+          worker_stats[job_klass] += 1
+        end
+
+        worker_stats.each do |class_name, count|
+          @metrics.add("sidekiq_running_jobs", count, name: class_name)
+        end
+
+        self
+      end
+
+      def probe_retries
+        return self unless connected?
+
+        retry_stats = Hash.new(0)
+
+        Sidekiq::RetrySet.new.map do |job|
+          retry_stats[job.klass] += 1
+        end
+
+        retry_stats.each do |class_name, count|
+          @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
+        end
+
+        self
+      end
+
+      def probe_dead
+        return self unless connected?
+
+        @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
+
+        self
+      end
+
+      def write_to(target)
+        target.write(@metrics.to_s)
+      end
+
+      private
+
+      def redis_options
+        options = {
+          url: @opts[:redis_url],
+          namespace: "resque:gitlab",
+          connect_timeout: 1,
+          reconnect_attempts: 0
+        }
+
+        options[:id] = nil unless redis_enable_client?
+        options
+      end
+
+      def redis_enable_client?
+        return true if @opts[:redis_enable_client].nil?
+
+        @opts[:redis_enable_client]
+      end
+
+      def connected?
+        @connected ||= begin
+                         Sidekiq.redis do |conn|
+                           conn.get("foo")
+                         end
+                         true
+                       end
+      rescue Redis::CannotConnectError, Redis::TimeoutError # rubocop:disable Lint/HandleExceptions
+        # Maybe we're trying connecting to a slave
+      end
+
+      def ensure_queue_job_stats_script_loaded
+        return unless connected?
+
+        Sidekiq.redis do |conn|
+          # Using administrative commands on conn directly (which is a Redis::Namespace)
+          # will be removed in redis-namespace 2.0.
+          next if conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
+          conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT)
+        end
+      end
+    end
+  end
+end