atatus 1.3.0 → 1.4.0

data/lib/atatus/metrics/breakdown_set.rb

@@ -0,0 +1,31 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Atatus
+  module Metrics
+    # @api private
+    class BreakdownSet < SpanScopedSet
+      def initialize(config)
+        super
+
+        disable! unless config.breakdown_metrics?
+      end
+    end
+  end
+end

data/lib/atatus/metrics/{cpu_mem.rb → cpu_mem_set.rb}

@@ -1,107 +1,161 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # frozen_string_literal: true
 
 module Atatus
   module Metrics
     # @api private
-    class CpuMem
+    class CpuMemSet < Set
       include Logging
 
       # @api private
       class Sample
         # rubocop:disable Metrics/ParameterLists
         def initialize(
+          page_size:,
+          process_cpu_usage:,
+          process_memory_rss:,
+          process_memory_size:,
           system_cpu_total:,
           system_cpu_usage:,
-          system_memory_total:,
           system_memory_free:,
-          process_cpu_usage:,
-          process_memory_size:,
-          process_memory_rss:,
-          page_size:
+          system_memory_total:
         )
+          @page_size = page_size
+          @process_cpu_usage = process_cpu_usage
+          @process_memory_rss = process_memory_rss
+          @process_memory_size = process_memory_size
           @system_cpu_total = system_cpu_total
           @system_cpu_usage = system_cpu_usage
-          @system_memory_total = system_memory_total
           @system_memory_free = system_memory_free
-          @process_cpu_usage = process_cpu_usage
-          @process_memory_size = process_memory_size
-          @process_memory_rss = process_memory_rss
-          @page_size = page_size
+          @system_memory_total = system_memory_total
         end
         # rubocop:enable Metrics/ParameterLists
 
-        attr_accessor :system_cpu_total, :system_cpu_usage,
-          :system_memory_total, :system_memory_free, :process_cpu_usage,
-          :process_memory_size, :process_memory_rss, :page_size
-
-        def delta(previous)
-          dup.tap do |sample|
-            sample.system_cpu_total =
-              system_cpu_total - previous.system_cpu_total
-            sample.system_cpu_usage =
-              system_cpu_usage - previous.system_cpu_usage
-            sample.process_cpu_usage =
-              process_cpu_usage - previous.process_cpu_usage
-          end
-        end
+        attr_accessor(
+          :page_size,
+          :process_cpu_usage,
+          :process_memory_rss,
+          :process_memory_size,
+          :system_cpu_total,
+          :system_cpu_usage,
+          :system_memory_free,
+          :system_memory_total
+        )
       end
 
       def initialize(config)
-        @config = config
+        @vmset_disabled = false
+        super
+
         @sampler = sampler_for_platform(Metrics.platform)
+        read! # set @previous on boot
       end
 
-      attr_reader :config, :sampler
+      attr_reader :config
 
-      def sample
-        @sampler.sample
+      def collect
+        read!
+        super
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      def collect
-        return unless sampler
+      private
+
+      def vmset_disable!
+        @vmset_disabled = true
+      end
 
-        current = sample
+      def vmset_disabled?
+        @vmset_disabled
+      end
+
+      def sampler_for_platform(platform)
+        case platform
+        when :linux then Linux.new
+        else
+          warn "Unsupported platform '#{platform}' - Disabling system metrics"
+          disable!
+          nil
+        end
+      end
+
+      def read!
+        return if disabled?
+
+        current = @sampler.sample
 
         unless @previous
           @previous = current
           return
         end
 
-        delta = current.delta(@previous)
+        cpu_usage_pct, cpu_process_pct = calculate_deltas(current, @previous)
 
-        cpu_usage_pct = delta.system_cpu_usage.to_f / delta.system_cpu_total
-        cpu_process_pct = delta.process_cpu_usage.to_f / delta.system_cpu_total
+        gauge(:'system.cpu.total.norm.pct').value = cpu_usage_pct
+        gauge(:'system.memory.actual.free').value = current.system_memory_free
+        gauge(:'system.memory.total').value = current.system_memory_total
+        gauge(:'system.process.cpu.total.norm.pct').value = cpu_process_pct
+        gauge(:'system.process.memory.size').value = current.process_memory_size
+        gauge(:'system.process.memory.rss.bytes').value =
+          current.process_memory_rss * current.page_size
 
         @previous = current
 
-        {
-          'system.cpu.total.norm.pct': cpu_usage_pct,
-          'system.memory.actual.free': current.system_memory_free,
-          'system.memory.total': current.system_memory_total,
-          'system.process.cpu.total.norm.pct': cpu_process_pct,
-          'system.process.memory.size': current.process_memory_size,
-          'system.process.memory.rss.bytes':
-            current.process_memory_rss * current.page_size
-        }
+        return if vmset_disabled?
+
+        stat = GC.stat
+
+        gauge(:'ruby.gc.count').value = stat[:count]
+        gauge(:'ruby.threads').value = Thread.list.count
+        gauge(:'ruby.heap.slots.live').value = stat[:heap_live_slots]
+
+        gauge(:'ruby.heap.slots.free').value = stat[:heap_free_slots]
+        gauge(:'ruby.heap.allocations.total').value =
+          stat[:total_allocated_objects]
+
+        return unless GC::Profiler.enabled?
+        @total_time ||= 0
+        @total_time += GC::Profiler.total_time
+        GC::Profiler.clear
+        gauge(:'ruby.gc.time').value = @total_time
+      rescue TypeError => e
+        error 'VM metrics encountered error: %s', e
+        debug('Backtrace:') { e.backtrace.join("\n") }
+
+        vmset_disable!
       end
-      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
-      private
+      def calculate_deltas(current, previous)
+        system_cpu_total =
+          current.system_cpu_total - previous.system_cpu_total
+        system_cpu_usage =
+          current.system_cpu_usage - previous.system_cpu_usage
+        process_cpu_usage =
+          current.process_cpu_usage - previous.process_cpu_usage
 
-      def sampler_for_platform(platform)
-        case platform
-        when :linux then Linux.new
-        else
-          warn "Unsupported platform '#{platform}' - Disabling metrics"
-          @disabled = true
-          nil
-        end
+        cpu_usage_pct = system_cpu_usage.to_f / system_cpu_total
+        cpu_process_pct = process_cpu_usage.to_f / system_cpu_total
+
+        [cpu_usage_pct, cpu_process_pct]
       end
 
       # @api private
       class Linux
-        # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
         def sample
           proc_stat = ProcStat.new.read!
           proc_self_stat = ProcSelfStat.new.read!
@@ -118,7 +172,6 @@ module Atatus
             page_size: meminfo.page_size
           )
         end
-        # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
         # @api private
         class ProcStat
@@ -136,8 +189,6 @@ module Atatus
             guest
             guest_nice
           ].freeze
-
-          # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
           def read!
             stat =
               IO.readlines('/proc/stat')
@@ -165,7 +216,6 @@ module Atatus
 
             self
           end
-          # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
         end
 
         UTIME_POS = 13
@@ -196,8 +246,6 @@ module Atatus
         # @api private
         class Meminfo
           attr_reader :total, :available, :page_size
-
-          # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
           # rubocop:disable Metrics/PerceivedComplexity
           # rubocop:disable Metrics/CyclomaticComplexity
           def read!
@@ -232,7 +280,6 @@ module Atatus
           end
           # rubocop:enable Metrics/CyclomaticComplexity
           # rubocop:enable Metrics/PerceivedComplexity
-          # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
         end
       end
     end

data/lib/atatus/metrics/metric.rb

@@ -0,0 +1,140 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Atatus
+  module Metrics
+    # @api private
+    class Metric
+      def initialize(
+        key,
+        initial_value: nil,
+        tags: nil,
+        reset_on_collect: false
+      )
+        @key = key
+        @initial_value = initial_value
+        @value = initial_value
+        @tags = tags
+        @reset_on_collect = reset_on_collect
+        @mutex = Mutex.new
+      end
+
+      attr_reader :key, :initial_value, :tags, :value
+
+      def value=(value)
+        @mutex.synchronize { @value = value }
+      end
+
+      def reset!
+        self.value = initial_value
+      end
+
+      def tags?
+        !!tags&.any?
+      end
+
+      def reset_on_collect?
+        @reset_on_collect
+      end
+
+      def collect
+        @mutex.synchronize do
+          collected = @value
+
+          @value = initial_value if reset_on_collect?
+
+          return nil if reset_on_collect? && collected == 0
+
+          collected
+        end
+      end
+    end
+
+    # @api private
+    class NoopMetric
+      def value; end
+
+      def value=(_); end
+
+      def collect; end
+
+      def reset!; end
+
+      def tags?; end
+
+      def reset_on_collect?; end
+
+      def inc!; end
+
+      def dec!; end
+
+      def update(_, delta: nil); end
+    end
+
+    # @api private
+    class Counter < Metric
+      def initialize(key, initial_value: 0, **args)
+        super(key, initial_value: initial_value, **args)
+      end
+
+      def inc!
+        @mutex.synchronize do
+          @value += 1
+        end
+      end
+
+      def dec!
+        @mutex.synchronize do
+          @value -= 1
+        end
+      end
+    end
+
+    # @api private
+    class Gauge < Metric
+      def initialize(key, **args)
+        super(key, initial_value: 0, **args)
+      end
+    end
+
+    # @api private
+    class Timer < Metric
+      def initialize(key, **args)
+        super(key, initial_value: 0, **args)
+        @count = 0
+      end
+
+      attr_accessor :count
+
+      def update(duration, delta: 0)
+        @mutex.synchronize do
+          @value += duration
+          @count += delta
+        end
+      end
+
+      def reset!
+        @mutex.synchronize do
+          @value = 0
+          @count = 0
+        end
+      end
+    end
+  end
+end

data/lib/atatus/metrics/set.rb

@@ -0,0 +1,123 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Atatus
+  # @api private
+  module Metrics
+    NOOP = NoopMetric.new
+
+    # @api private
+    class Set
+      include Logging
+
+      DISTINCT_LABEL_LIMIT = 1000
+
+      def initialize(config)
+        @config = config
+        @metrics = {}
+        @disabled = false
+        @lock = Mutex.new
+      end
+
+      attr_reader :metrics
+
+      def disable!
+        @disabled = true
+      end
+
+      def disabled?
+        @disabled
+      end
+
+      def counter(key, tags: nil, **args)
+        metric(Counter, key, tags: tags, **args)
+      end
+
+      def gauge(key, tags: nil, **args)
+        metric(Gauge, key, tags: tags, **args)
+      end
+
+      def timer(key, tags: nil, **args)
+        metric(Timer, key, tags: tags, **args)
+      end
+
+      def metric(kls, key, tags: nil, **args)
+        if @config.disable_metrics.any? { |p| p.match? key }
+          return NOOP
+        end
+
+        key = key_with_tags(key, tags)
+        return metrics[key] if metrics[key]
+
+        @lock.synchronize do
+          return metrics[key] if metrics[key]
+
+          metrics[key] =
+            if metrics.length < DISTINCT_LABEL_LIMIT
+              kls.new(key, tags: tags, **args)
+            else
+              unless @label_limit_logged
+                warn(
+                  'The limit of %d metricsets has been reached, no new ' \
+                   'metricsets will be created.', DISTINCT_LABEL_LIMIT
+                )
+                @label_limit_logged = true
+              end
+
+              NOOP
+            end
+        end
+      end
+
+      def collect
+        return if disabled?
+
+        @lock.synchronize do
+          metrics.each_with_object({}) do |(key, metric), sets|
+            next unless (value = metric.collect)
+
+            # metrics have a key of name and flat array of key-value pairs
+            #   eg [name, key, value, key, value]
+            # they can be sent in the same metricset but not if they have
+            # differing tags. So, we split the resulting sets by tags first.
+            name, *tags = key
+            sets[tags] ||= Metricset.new
+
+            # then we set the `samples` value for the metricset
+            set = sets[tags]
+            set.samples[name] = value
+
+            # and finally we copy the tags from the Metric to the Metricset
+            set.merge_tags! metric.tags
+          end.values
+        end
+      end
+
+      private
+
+      def key_with_tags(key, tags)
+        return key unless tags
+
+        tuple = tags.keys.zip(tags.values)
+        tuple.flatten!
+        tuple.unshift(key)
+      end
+    end
+  end
+end