gitlab-monitor 4.0.0

data/lib/gitlab_monitor/database/remote_mirrors.rb

@@ -0,0 +1,74 @@
+module GitLab
+  module Monitor
+    module Database
+      # A helper class to collect remote mirrors metrics.
+      class RemoteMirrorsCollector < Base
+        QUERY = <<~SQL.freeze
+          SELECT project_id, url,
+          EXTRACT(EPOCH FROM last_successful_update_at) AS last_successful_update_at,
+          EXTRACT(EPOCH FROM last_update_at) AS last_update_at
+          FROM remote_mirrors WHERE project_id IN (%s) AND enabled = 't'
+        SQL
+
+        def initialize(args)
+          super(args)
+
+          @project_ids = args[:project_ids]
+        end
+
+        def run
+          return if @project_ids.nil? || @project_ids.empty?
+
+          execute(QUERY % [@project_ids.join(",")]) # rubocop:disable Style/FormatString
+        end
+
+        private
+
+        def execute(query)
+          with_connection_pool do |conn|
+            conn.exec(query)
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          nil
+        end
+      end
+
+      # The prober which is called when gathering metrics
+      class RemoteMirrorsProber
+        def initialize(opts, metrics: PrometheusMetrics.new)
+          @metrics = metrics
+          @collector = RemoteMirrorsCollector.new(
+            connection_string: opts[:connection_string],
+            project_ids: opts[:project_ids]
+          )
+        end
+
+        def probe_db # rubocop:disable Metrics/MethodLength
+          results = @collector.run
+          results.to_a.each do |row|
+            @metrics.add(
+              "project_remote_mirror_last_successful_update_time_seconds",
+              row["last_successful_update_at"].to_i,
+              project_id: row["project_id"],
+              url: row["url"]
+            )
+            @metrics.add(
+              "project_remote_mirror_last_update_time_seconds",
+              row["last_update_at"].to_i,
+              project_id: row["project_id"],
+              url: row["url"]
+            )
+          end
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+
+        def write_to(target)
+          target.write(@metrics.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/database/row_count.rb

@@ -0,0 +1,164 @@
+require "set"
+
+module GitLab
+  module Monitor
+    module Database
+      # A helper class that executes the query its given and returns an int of
+      # the row count
+      # This class works under the assumption you do COUNT(*) queries, define
+      # queries in the QUERIES constant. If in doubt how these work, read
+      # #construct_query
+      class RowCountCollector < Base
+        WHERE_MIRROR_ENABLED = <<~SQL.freeze
+          projects.mirror = true
+          AND project_mirror_data.retry_count <= 14
+          AND (projects.visibility_level = 20 OR plans.name IN ('early_adopter', 'bronze', 'silver', 'gold'))
+        SQL
+
+        MIRROR_QUERY = {
+          select: :projects,
+          joins:  <<~SQL,
+            INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
+            INNER JOIN namespaces ON projects.namespace_id = namespaces.id
+            LEFT JOIN plans ON namespaces.plan_id = plans.id
+          SQL
+          check: "SELECT 1 FROM information_schema.tables WHERE table_name='plans'"
+        }.freeze
+
+        QUERIES = {
+          mirrors_ready_to_sync: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status NOT IN ('scheduled', 'started')
+              AND project_mirror_data.next_execution_timestamp <= NOW()
+            SQL
+          ),
+          mirrors_not_updated_recently: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status NOT IN ('scheduled', 'started')
+              AND (project_mirror_data.next_execution_timestamp - project_mirror_data.last_update_at) <= '30 minutes'::interval
+              AND project_mirror_data.last_update_at < NOW() - '30 minutes'::interval
+            SQL
+          ),
+          mirrors_updated_very_recently: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status NOT IN ('scheduled', 'started')
+              AND project_mirror_data.last_update_at >= NOW() - '30 seconds'::interval
+            SQL
+          ),
+          mirrors_behind_schedule: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status NOT IN ('scheduled', 'started')
+              AND project_mirror_data.next_execution_timestamp <= NOW() - '10 seconds'::interval
+            SQL
+          ),
+          mirrors_scheduled_or_started: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status IN ('scheduled', 'started')
+            SQL
+          ),
+          mirrors_scheduled: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status = 'scheduled'
+            SQL
+          ),
+          mirrors_started: MIRROR_QUERY.merge( # EE only
+            where: <<~SQL
+              #{WHERE_MIRROR_ENABLED}
+              AND project_mirror_data.status = 'started'
+            SQL
+          ),
+          soft_deleted_projects: { select: :projects, where: "pending_delete=true" },
+          orphaned_projects: {
+            select: :projects,
+            joins: "LEFT JOIN namespaces ON projects.namespace_id = namespaces.id",
+            where: "namespaces.id IS NULL"
+          },
+          uploads: { select: :uploads }
+        }.freeze
+
+        def initialize(args)
+          super(args)
+
+          @selected_queries = Set.new(args[:selected_queries].map(&:to_sym)) unless args[:selected_queries].nil?
+        end
+
+        def run
+          results = Hash.new(0)
+
+          QUERIES.each do |key, query_hash|
+            next if query_hash[:check] && !successful_check?(query_hash[:check])
+            next if !@selected_queries.nil? && !@selected_queries.include?(key)
+
+            results[key] = count_from_query_hash(query_hash)
+          end
+
+          results
+        end
+
+        private
+
+        def count_from_query_hash(query_hash)
+          result = execute(construct_query(query_hash))
+          return 0 unless result
+
+          result[0]["count"]
+        end
+
+        def successful_check?(query)
+          result = execute("SELECT EXISTS (#{query})")
+          return unless result
+
+          result[0]["exists"] == "t"
+        end
+
+        def execute(query)
+          with_connection_pool do |conn|
+            conn.exec(query)
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          nil
+        end
+
+        # Not private so I can test it without meta programming tricks
+        def construct_query(query)
+          query_string = "SELECT COUNT(*) FROM #{query[:select]} "
+          query_string << "#{query[:joins]} "       if query[:joins]
+          query_string << "WHERE #{query[:where]}"  if query[:where]
+          query_string << ";"
+        end
+      end
+
+      # The prober which is called when gathering metrics
+      class RowCountProber
+        def initialize(opts, metrics: PrometheusMetrics.new)
+          @metrics = metrics
+          @collector = RowCountCollector.new(
+            connection_string: opts[:connection_string],
+            selected_queries: opts[:selected_queries]
+          )
+        end
+
+        def probe_db
+          results = @collector.run
+          results.each do |key, value|
+            @metrics.add("gitlab_database_rows", value.to_i, query_name: key.to_s)
+          end
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+
+        def write_to(target)
+          target.write(@metrics.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/database/tuple_stats.rb

@@ -0,0 +1,53 @@
+module GitLab
+  module Monitor
+    module Database
+      # A helper class to collect tuple stats from the database
+      #
+      # It takes a connection string (e.g. "dbname=test port=5432")
+      class TupleStatsCollector < Base
+        COLUMNS = %w(relname seq_tup_read idx_tup_fetch n_tup_ins n_tup_upd n_tup_del n_tup_hot_upd n_dead_tup seq_scan)
+                  .join(",")
+        QUERY   = <<-SQL.freeze
+          SELECT #{COLUMNS}
+          FROM pg_stat_user_tables
+          WHERE relname IN (SELECT tablename FROM pg_tables WHERE tableowner = 'gitlab')
+          GROUP BY #{COLUMNS}
+        SQL
+
+        def run
+          with_connection_pool do |conn|
+            conn.exec(QUERY).each.with_object({}) do |row, stats|
+              stats[row.delete("relname")] = row
+            end
+          end
+        end
+      end
+
+      # Probes the DB specified by opts[:connection_string] for tuple stats, then converts them to metrics
+      class TuplesProber
+        def initialize(opts, metrics: PrometheusMetrics.new)
+          @metrics   = metrics
+          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string])
+        end
+
+        def probe_db
+          result = @collector.run
+
+          result.each do |table_name, tuple_stats|
+            tuple_stats.each do |column_name, value|
+              @metrics.add("gitlab_database_stat_table_#{column_name}", value.to_s, table_name: table_name)
+            end
+          end
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+
+        def write_to(target)
+          target.write(@metrics.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/git.rb

@@ -0,0 +1,144 @@
+require "open3"
+
+module GitLab
+  module Monitor
+    # Git monitoring helping class
+    #
+    # Takes a repository path for construction and provides 2 main methods:
+    #   - pull
+    #   - push
+    #
+    # Both methods return a CommandResult which includes the output of the execution
+    # plus the tracked execution time.
+    class Git
+      def initialize(repo)
+        fail "Repository #{repo} does not exists" unless Dir.exist? repo
+        @repo = repo
+        @tracker = TimeTracker.new
+      end
+
+      def pull
+        @tracker.track { execute "git pull -q" }
+      end
+
+      def push
+        empty_commit
+        @tracker.track { execute "git push -q" }
+      end
+
+      def empty_commit(message = "Beep")
+        @tracker.track { execute("git commit --allow-empty -m '#{message}'") }
+      end
+
+      private
+
+      def execute(command)
+        result = CommandResult.new(*Open3.capture2e(command, chdir: @repo))
+        fail "Command #{command} failed with status #{result.status}\n#{result.stdout}" if result.failed?
+        result
+      end
+    end
+
+    # Result of a command
+    #
+    # Provides some handy methods for checking if the execution failed and a simple to_s that will
+    # return the command output
+    CommandResult = Struct.new(:stdout, :status) do
+      def failed?
+        status.nonzero?
+      end
+
+      def status
+        self[:status].exitstatus
+      end
+
+      def to_s
+        stdout
+      end
+    end
+
+    # Handles creating a Git object, probing for both pull and push, and finally writing to metrics
+    #
+    # Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
+    # metrics writer to something else.
+    class GitProber
+      def initialize(opts, metrics: PrometheusMetrics.new)
+        @metrics = metrics
+        @labels = opts[:labels] || {}
+        @git = Git.new(opts[:source])
+      end
+
+      def probe_pull
+        @metrics.add "git_pull_time_milliseconds", (@git.pull.time * 1000).to_i, **@labels
+        self
+      end
+
+      def probe_push
+        @metrics.add "git_push_time_milliseconds", (@git.push.time * 1000).to_i, **@labels
+        self
+      end
+
+      def write_to(target)
+        target.write(@metrics.to_s)
+      end
+    end
+
+    # A special prober for git processes
+    class GitProcessProber
+      def initialize(opts, metrics: PrometheusMetrics.new)
+        @opts    = opts
+        @metrics = metrics
+      end
+
+      def probe_git # rubocop:disable Metrics/MethodLength
+        counts = Hash.new(0)
+
+        Utils.pgrep("^git ").each do |pid|
+          process_cmd = begin
+                          File.read("/proc/#{pid}/cmdline")
+                        rescue
+                          "" # Process file is gone (race condition)
+                        end
+          subcommand  = self.class.extract_subcommand(process_cmd)
+          next unless subcommand # Unlikely, but just to be safe
+
+          name = "git #{subcommand}"
+          counts[name] += 1
+
+          prober = ProcessProber.new(
+            {
+              name: name,
+              pid_or_pattern: pid,
+              quantiles: @opts[:quantiles]
+            },
+            metrics: @metrics
+          )
+
+          prober
+            .probe_stat
+        end
+
+        counts.each do |name, count|
+          @metrics.add("process_count", count, name: name)
+        end
+
+        self
+      end
+
+      def write_to(target)
+        target.write(@metrics.to_s)
+      end
+
+      def self.extract_subcommand(cmd)
+        return if cmd.empty?
+        cmd_splitted = cmd.split("\u0000") # cmdline does not return it space-separated
+
+        cmd_splitted.shift # Because it's "git"
+        cmd_splitted.shift while cmd_splitted.first &&
+                                 (cmd_splitted.first.empty? || cmd_splitted.first !~ /^[^-][a-z\-]*$/)
+
+        cmd_splitted[0]
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/memstats.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative "memstats/mapping"
+
+# Ported from https://github.com/discourse/discourse/blob/master/script/memstats.rb
+#
+# Aggregate Print useful information from /proc/[pid]/smaps
+#
+# pss  - Roughly the amount of memory that is "really" being used by the pid
+# swap - Amount of swap this process is currently using
+#
+# Reference:
+#  http://www.mjmwired.net/kernel/Documentation/filesystems/proc.txt#361
+#
+# Example:
+#   # ./memstats.rb 4386
+#   Process:             4386
+#   Command Line:        /usr/bin/mongod -f /etc/mongo/mongod.conf
+#   Memory Summary:
+#     private_clean             107,132 kB
+#     private_dirty           2,020,676 kB
+#     pss                     2,127,860 kB
+#     rss                     2,128,536 kB
+#     shared_clean                  728 kB
+#     shared_dirty                    0 kB
+#     size                  149,281,668 kB
+#     swap                    1,719,792 kB
+module GitLab
+  module Monitor
+    module MemStats
+      # Aggregates all metrics for a single PID in /proc/<pid>/smaps
+      class Aggregator
+        attr_accessor :pid, :totals
+
+        def initialize(pid)
+          @pid = pid
+          @totals = Hash.new(0)
+          @mappings = []
+          @valid = true
+
+          populate_info
+        end
+
+        def valid?
+          @valid
+        end
+
+        private
+
+        attr_accessor :mappings
+
+        def consume_mapping(map_lines, totals)
+          m = Mapping.new(map_lines)
+
+          Mapping::FIELDS.each do |field|
+            totals[field] += m.send(field)
+          end
+
+          m
+        end
+
+        def create_memstats_not_available(totals)
+          Mapping::FIELDS.each do |field|
+            totals[field] += Float::NAN
+          end
+        end
+
+        def populate_info # rubocop:disable Metrics/MethodLength
+          File.open("/proc/#{@pid}/smaps") do |smaps|
+            map_lines = []
+
+            loop do
+              break if smaps.eof?
+
+              line = smaps.readline.strip
+
+              case line
+              when /\w+:\s+/
+                map_lines << line
+              when /[0-9a-f]+:[0-9a-f]+\s+/
+                mappings << consume_mapping(map_lines, totals) if map_lines.size.positive?
+
+                map_lines.clear
+                map_lines << line
+              else
+                break
+              end
+            end
+          end
+        rescue => e
+          puts "Error: #{e}"
+          @valid = false
+          create_memstats_not_available(totals)
+        end
+      end
+    end
+  end
+end