logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/lib/logstash/instrument/metric_type.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/instrument/metric_type/counter"
+require "logstash/instrument/metric_type/mean"
+require "logstash/instrument/metric_type/gauge"
+
+module LogStash module Instrument
+  module MetricType
+    METRIC_TYPE_LIST = {
+      :counter => LogStash::Instrument::MetricType::Counter,
+      :mean => LogStash::Instrument::MetricType::Mean,
+      :gauge => LogStash::Instrument::MetricType::Gauge
+    }.freeze
+
+    # Use the string to generate a concrete class for this metrics
+    #
+    # @param [String] The name of the class
+    # @param [Array] Namespaces list
+    # @param [String] The metric key
+    # @raise [NameError] If the class is not found
+    def self.create(type, namespaces, key)
+      METRIC_TYPE_LIST[type].new(namespaces, key)
+    end
+  end
+end; end

data/lib/logstash/instrument/metric_type/base.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/util"
+
+module LogStash module Instrument module MetricType
+  class Base
+    attr_reader :namespaces, :key
+
+    def initialize(namespaces, key)
+      @namespaces = namespaces
+      @key = key
+    end
+
+    def inspect
+      "#{self.class.name} - namespaces: #{namespaces} key: #{key} value: #{value}"
+    end
+
+    def to_hash
+      {
+        "namespaces" => namespaces,
+        "key" => key,
+        "type" => type,
+        "value" => value
+      }
+    end
+
+    def to_json_data
+      value
+    end
+
+    def type
+      @type ||= LogStash::Util.class_name(self).downcase
+    end
+  end
+end; end; end

data/lib/logstash/instrument/metric_type/counter.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "logstash/instrument/metric_type/base"
+require "concurrent"
+
+module LogStash module Instrument module MetricType
+  class Counter < Base
+    def initialize(namespaces, key, value = 0)
+      super(namespaces, key)
+
+      @counter = Concurrent::AtomicFixnum.new(value)
+    end
+
+    def increment(value = 1)
+      @counter.increment(value)
+    end
+
+    def decrement(value = 1)
+      @counter.decrement(value)
+    end
+
+    def execute(action, value = 1)
+      @counter.send(action, value)
+    end
+
+    def value
+      @counter.value
+    end
+  end
+end; end; end

data/lib/logstash/instrument/metric_type/gauge.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/instrument/metric_type/base"
+require "concurrent/atomic_reference/mutex_atomic"
+require "logstash/json"
+
+module LogStash module Instrument module MetricType
+  class Gauge < Base
+    def initialize(namespaces, key)
+      super(namespaces, key)
+
+      @gauge = Concurrent::MutexAtomicReference.new()
+    end
+
+    def execute(action, value = nil)
+      @gauge.set(value)
+    end
+
+    def value
+      @gauge.get
+    end
+  end
+end; end; end

data/lib/logstash/instrument/metric_type/mean.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "logstash/instrument/metric_type/base"
+require "concurrent"
+
+module LogStash module Instrument module MetricType
+  class Mean < Base
+    def initialize(namespaces, key)
+      super(namespaces, key)
+
+      @counter = Concurrent::AtomicFixnum.new
+      @sum = Concurrent::AtomicFixnum.new
+    end
+
+    def increment(value = 1)
+      @counter.increment
+      @sum.increment(value)
+    end
+
+    def decrement(value = 1)
+      @counter.decrement
+      @sum.decrement(value)
+    end
+
+    def mean
+      if @counter > 0
+        @sum.value / @counter.value
+      else
+        0
+      end
+    end
+    alias_method :value, :mean
+  end
+end; end; end

data/lib/logstash/instrument/namespaced_metric.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require "logstash/instrument/metric"
+
+module LogStash module Instrument
+  # This class acts a a proxy between the metric library and the user calls.
+  #
+  # This is the class that plugins authors will use to interact with the `MetricStore`
+  # It has the same public interface as `Metric` class but doesnt require to send
+  # the namespace on every call.
+  #
+  # @see Logstash::Instrument::Metric
+  class NamespacedMetric
+    attr_reader :namespace_name
+    # Create metric with a specific namespace
+    #
+    # @param metric [LogStash::Instrument::Metric] The metric instance to proxy
+    # @param namespace [Array] The namespace to use
+    def initialize(metric, namespace_name)
+      @metric = metric
+      @namespace_name = Array(namespace_name)
+    end
+
+    def increment(key, value = 1)
+      @metric.increment(namespace_name, key, value)
+    end
+
+    def decrement(namespace, key, value = 1)
+      @metric.decrement(namespace_name, key, value)
+    end
+
+    def gauge(key, value)
+      @metric.gauge(namespace_name, key, value)
+    end
+
+    def report_time(key, duration)
+      @metric.report_time(namespace_name, key, duration)
+    end
+
+    def time(key, &block)
+      @metric.time(namespace_name, key, &block)
+    end
+
+    def collector
+      @metric.collector
+    end
+
+    def namespace(name)
+      NamespacedMetric.new(metric, namespace_name.concat(Array(name)))
+    end
+
+    private
+    attr_reader :metric
+  end
+end; end

data/lib/logstash/instrument/null_metric.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash/instrument/metric"
 
 module LogStash module Instrument
  # This class is used in the context when we disable the metric collection
@@ -10,7 +11,7 @@ module LogStash module Instrument
    def increment(key, value = 1)
    end
 
-   def decrement(key, value = 1)
+   def decrement(namespace, key, value = 1)
    end
 
    def gauge(key, value)
@@ -21,7 +22,7 @@ module LogStash module Instrument
 
    # We have to manually redefine this method since it can return an
    # object this object also has to be implemented as a NullObject
-   def time(key, &block)
+   def time(key)
      if block_given?
        yield
      else
@@ -29,7 +30,7 @@ module LogStash module Instrument
      end
    end
 
-   def namespace(name)
+   def namespace(key)
      self.class.new
    end
 

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require "logstash/util"
+require "concurrent"
+
+module LogStash module Instrument module PeriodicPoller
+  class Base
+    include LogStash::Util::Loggable
+
+    DEFAULT_OPTIONS = {
+      :polling_interval => 1,
+      :polling_timeout => 60
+    }
+
+    public
+    def initialize(metric, options = {})
+      @metric = metric
+      @options = DEFAULT_OPTIONS.merge(options)
+      configure_task
+    end
+
+    def update(time, result, exception)
+      return unless exception
+
+      logger.error("PeriodicPoller: exception",
+                   :poller => self,
+                   :result => result,
+                   :exception => exception,
+                   :executed_at => time)
+    end
+
+    def collect
+      raise NotImplementedError, "#{self.class.name} need to implement `#collect`"
+    end
+
+    def start
+      logger.debug("PeriodicPoller: Starting",
+                   :polling_interval => @options[:polling_interval],
+                   :polling_timeout => @options[:polling_timeout]) if logger.debug?
+      @task.execute
+    end
+
+    def stop
+      logger.debug("PeriodicPoller: Stopping")
+      @task.shutdown
+    end
+
+    protected
+    def configure_task
+      @task = Concurrent::TimerTask.new { collect }
+      @task.execution_interval = @options[:polling_interval]
+      @task.timeout_interval = @options[:polling_timeout]
+      @task.add_observer(self)
+    end
+  end
+end
+end; end

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/base"
+require 'monitoring'
+
+module LogStash module Instrument module PeriodicPoller
+  class JVM < Base
+
+    attr_reader :metric
+
+    def initialize(metric, options = {})
+      super(metric, options)
+      @metric = metric
+    end
+
+    def collect
+      raw = JRMonitor.memory.generate
+      collect_heap_metrics(raw)
+      collect_non_heap_metrics(raw)
+      collect_pools_metrics(raw)
+    end
+
+    private
+
+    def collect_heap_metrics(data)
+      heap = aggregate_information_for(data["heap"].values)
+      heap[:used_percent] = (heap[:used_in_bytes] / heap[:max_in_bytes].to_f)*100.0
+
+      heap.each_pair do |key, value|
+        metric.gauge([:jvm, :memory, :heap], key, value.to_i)
+      end
+    end
+
+    def collect_non_heap_metrics(data)
+      non_heap = aggregate_information_for(data["non_heap"].values)
+      non_heap.each_pair do |key, value|
+        metric.gauge([:jvm, :memory, :non_heap],key, value.to_i)
+      end
+    end
+
+    def collect_pools_metrics(data)
+      metrics = build_pools_metrics(data)
+      metrics.each_pair do |key, hash|
+        hash.each_pair do |p,v|
+          metric.gauge([:jvm, :memory, :pools, key.to_sym], p, v)
+        end
+      end
+    end
+
+    def build_pools_metrics(data)
+      heap = data["heap"]
+      old  = {}
+      old = old.merge!(heap["CMS Old Gen"]) if heap.has_key?("CMS Old Gen")
+      old = old.merge!(heap["PS Old Gen"])  if heap.has_key?("PS Old Gen")
+      young = {}
+      young = young.merge!(heap["Par Eden Space"]) if heap.has_key?("Par Eden Space")
+      young = young.merge!(heap["PS Eden Space"])  if heap.has_key?("PS Eden Space")
+      survivor = {}
+      survivor = survivor.merge!(heap["Par Survivor Space"]) if heap.has_key?("Par Survivor Space")
+      survivor = survivor.merge!(heap["PS Survivor Space"])  if heap.has_key?("PS Survivor Space")
+      {
+        "young"    => aggregate_information_for(young),
+        "old"      => aggregate_information_for(old),
+        "survivor" => aggregate_information_for(survivor)
+      }
+    end
+
+    def aggregate_information_for(collection)
+      collection.reduce(default_information_accumulator) do |m,e|
+        e = { e[0] => e[1] } if e.is_a?(Array)
+        e.each_pair do |k,v|
+          m[:used_in_bytes] += v       if k.include?("used")
+          m[:committed_in_bytes] += v  if k.include?("committed")
+          m[:max_in_bytes] += v        if k.include?("max")
+          m[:peak_max_in_bytes] += v   if k.include?("peak.max")
+          m[:peak_used_in_bytes] += v  if k.include?("peak.used")
+        end
+        m
+      end
+    end
+
+    def default_information_accumulator
+      {
+        :used_in_bytes => 0,
+        :committed_in_bytes => 0,
+        :max_in_bytes => 0,
+        :peak_used_in_bytes => 0,
+        :peak_max_in_bytes  => 0
+      }
+    end
+
+  end
+end; end; end

data/lib/logstash/instrument/periodic_poller/os.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/base"
+
+module LogStash module Instrument module PeriodicPoller
+  class Os < Base
+    def initialize(metric, options = {})
+      super(metric, options)
+    end
+
+    def collect
+    end
+  end
+end; end; end

data/lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+module LogStash module Instrument module PeriodicPoller
+  class PeriodicPollerObserver
+    include LogStash::Util::Loggable
+    
+    def initialize(poller)
+      @poller = poller
+    end
+
+    def update(time, result, exception)
+      if exception
+        logger.error("PeriodicPoller exception", :poller => @poller,
+                     :result => result,
+                     :exception => exception,
+                     :executed_at => time)
+      end
+    end
+  end
+end; end; end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/os"
+require "logstash/instrument/periodic_poller/jvm"
+
+module LogStash module Instrument
+  # Each PeriodPoller manager his own thread to do the poller
+  # of the stats, this class encapsulate the starting and stopping of the poller
+  # if the unique timer uses too much resource we can refactor this behavior here.
+  class PeriodicPollers
+    attr_reader :metric
+
+    def initialize(metric)
+      @metric = metric
+      @periodic_pollers = [PeriodicPoller::Os.new(metric),
+                          PeriodicPoller::JVM.new(metric)]
+    end
+
+    def start
+      @periodic_pollers.map(&:start)
+    end
+
+    def stop
+      @periodic_pollers.map(&:stop)
+    end
+  end
+end; end