stackify-api-ruby 1.0.0

data/lib/stackify/logs_sender.rb

@@ -0,0 +1,78 @@
+module Stackify
+  class LogsSender < HttpClient
+
+    LOGS_URI = URI('https://dev.stackify.com/api/Log/Save')
+
+    def start
+      worker = Stackify::Worker.new 'Main sending thread'
+      task = Stackify::ScheduleTask.new do
+        send_logs
+      end
+      worker.async_perform Stackify.configuration.send_interval, task
+      Stackify.internal_log :debug, 'LogsSender: main sending thread is started'
+    end
+
+    def send_remained_msgs
+      if Stackify.working?
+        Stackify.internal_log :warn, 'Sending of remained msgs is possible when Stackify is terminating work.'
+      else
+        worker = Stackify::Worker.new 'RemainedJob worker'
+        task = send_all_remained_msgs_task Stackify.msgs_queue.pop_all
+        worker.perform 2, task
+      end
+    end
+
+    private
+
+    def send_logs attempts = 3
+      msgs = Stackify.msgs_queue.pop #it should wait until queue will get a new chunk if queque is empty
+      worker = Stackify::LogsSenderWorker.new
+      task = send_logs_task attempts, msgs
+      worker.async_perform 5, task
+    end
+
+    def properties
+      {
+        success_condition: lambda { |result| result.try(:code) == '200' },
+        limit: 1
+      }.dup
+    end
+
+    def send_logs_task attempts = nil, msgs
+      properties[:attempts] = attempts if attempts
+      Stackify::ScheduleTask.new properties do
+        failure_msg = 'LogsSender: tried to send logs'
+        Stackify.if_not_authorized failure_msg do
+          Stackify.internal_log :info, 'LogsSender: trying to send logs to Stackify...'
+          send_request LOGS_URI, gather_and_pack_data(msgs).to_json
+        end
+      end
+    end
+
+    def send_all_remained_msgs_task msgs
+      Stackify::ScheduleTask.new properties do
+        failure_msg = 'LogsSender: tried to remained send logs'
+        Stackify.if_not_authorized failure_msg do
+          Stackify.internal_log :info, 'LogsSender: trying to send remained logs to Stackify...'
+          send_request LOGS_URI, gather_and_pack_data(msgs).to_json
+        end
+      end
+    end
+
+    def gather_and_pack_data msgs
+      details = Stackify::EnvDetails.instance.auth_info
+      {
+        'CDID' => details['DeviceID'],
+        'CDAppID' => details['DeviceAppID'],
+        'Logger' => 'Rails logger',
+        'AppName' => details['AppName'],
+        'AppNameID' => details['AppNameID'],
+        'EnvID' => details['EnvID'],
+        'ServerName' => details['DeviceName'],
+        'Msgs' => msgs,
+        'AppLoc' => details['AppLocation'],
+        'Platform' => 'Ruby'
+      }
+    end
+  end
+end

data/lib/stackify/metrics/metric.rb

@@ -0,0 +1,68 @@
+require 'date'
+module Stackify::Metrics
+  class Metric
+    attr_accessor :category, :name, :metric_type, :is_increment,
+                  :settings, :value, :aggregate_key, :occurred
+
+    def initialize category, name, metric_type, metric_settings = nil
+      @category = category
+      @name = name
+      @metric_type = metric_type
+      @occurred = Time.now.utc
+      @occurred = get_rounded_time
+      @is_increment = false
+      @settings = metric_settings || MetricSettings.new
+    end
+
+    def calc_and_set_aggregate_key
+      @aggregate_key = @category.downcase + '-' + (@name || 'Missing Name').downcase +
+                       '-' + @metric_type.to_s + '-' + get_rounded_time.to_s
+    end
+
+    def calc_name_key
+      @category.downcase + '-' + (@name || 'Missing Name').downcase + '-' + @metric_type.to_s
+    end
+
+    def get_rounded_time
+      @occurred - @occurred.sec
+    end
+  end
+
+  class MetricSettings
+    attr_reader :autoreport_zero_if_nothing_reported,
+                :autoreport_last_value_if_nothing_reported
+
+    def autoreport_last_value_if_nothing_reported= value
+      @autoreport_last_value_if_nothing_reported = value
+      @autoreport_zero_if_nothing_reported = false if value
+    end
+
+    def autoreport_zero_if_nothing_reported= value
+      @autoreport_zero_if_nothing_reported = value
+      @autoreport_last_value_if_nothing_reported = false if value
+    end
+  end
+
+  class MetricForSubmit
+    attr_accessor :monitor_id, :value, :count, :occurred_utc, :monitor_type_id
+
+    def initialize metric
+      @value = metric.value.round 2
+      @monitor_id = metric.monitor_id || 0
+      @occurred_utc = metric.occurred_utc
+      @count = metric.count
+      @monitor_type_id = metric.metric_type
+    end
+
+    def to_h
+      {
+        'Value' => @value,
+        'MonitorID' => @monitor_id,
+        'OccurredUtc' => DateTime.parse(@occurred_utc.to_s).iso8601,
+        'Count' => @count,
+        'MonitorTypeID' => @monitor_type_id
+      }
+    end
+  end
+
+end

data/lib/stackify/metrics/metric_aggregate.rb

@@ -0,0 +1,52 @@
+module Stackify::Metrics
+  class MetricAggregate
+
+    attr_accessor :name, :category, :value, :count, :occurred_utc,
+                  :monitor_id, :metric_type, :name_key
+
+    def initialize metric
+      @name = metric.name
+      @category = metric.category
+      @metric_type = metric.metric_type
+      @value = 0
+      @count = 0
+      @occurred_utc = metric.get_rounded_time
+      @name_key = metric.calc_name_key
+    end
+
+    def aggregate_key
+      (@category || 'Missing Category').downcase + '-' + (@name || 'Missing Name').downcase +
+      '-' + @metric_type.to_s + '-' + @occurred_utc.to_s
+    end
+
+  end
+
+  class LatestAggregate
+    attr_accessor :category, :name, :metric_id, :occurred_utc,
+                  :value, :count, :metric_type
+
+    def initialize aggr_metric
+      @count = aggr_metric.count
+      @metric_type = aggr_metric.metric_type
+      @metric_id = aggr_metric.monitor_id
+      @name = aggr_metric.name
+      @occurred_utc = aggr_metric.occurred_utc
+      @value = aggr_metric.value
+      @count = aggr_metric.count
+      @category = aggr_metric.category
+    end
+
+    def to_h
+      {
+        'Count' => @count,
+        'MetricType' => @metric_type,
+        'MetricID' => @metric_id,
+        'Name' => @name,
+        'OccurredUtc' => @occurred_utc,
+        'Value' => @value,
+        'Count' => @count,
+        'Category' => @category
+      }
+    end
+  end
+end

data/lib/stackify/metrics/metrics.rb

@@ -0,0 +1,88 @@
+module Stackify
+  module Metrics
+    require_relative 'metrics_queue'
+    require_relative 'metric'
+    require_relative 'metric_aggregate'
+    require_relative 'monitor'
+    require_relative 'metrics_client'
+    require_relative 'metrics_sender'
+
+    METRIC_TYPES = {
+      metric_last: 134,
+      counter: 129,
+      metric_average: 132,
+      counter_time: 131
+    }
+    class << self
+      def metrics_client
+        @@metrics_client ||= Stackify::Metrics::MetricsClient.new
+      end
+
+      def average category, metric_name, value, advanced_settings = nil
+        m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:metric_average]
+        m.value = value
+        m.settings = advanced_settings
+        Stackify::Metrics.metrics_client.queue_metric m
+      end
+
+      def count category, metric_name, increment_by= 1, advanced_settings = nil
+        m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:counter]
+        m.value = increment_by
+        m.settings = advanced_settings
+        Stackify::Metrics.metrics_client.queue_metric m
+      end
+
+      def get_latest category, metric_name
+        metrics_client.get_latest category, metric_name
+      end
+
+      def get_latest_all_metrics
+        metrics_client.get_latest_all_metrics
+      end
+
+      def increment_gauge category, metric_name, increment_by = 1, advanced_settings = nil
+        m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:metric_last]
+        m.value = increment_by
+        m.is_increment = true
+        m.settings = advanced_settings
+        Stackify::Metrics.metrics_client.queue_metric m
+      end
+
+      def set_gauge category, metric_name, value, advanced_settings = nil
+        m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:metric_last]
+        m.value = value
+        m.settings = advanced_settings
+        Stackify::Metrics.metrics_client.queue_metric m
+      end
+
+      def sum category, metric_name, value, advanced_settings = nil
+        m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:counter]
+        m.value = value
+        m.settings = advanced_settings
+        Stackify::Metrics.metrics_client.queue_metric m
+      end
+
+      def time category, metric_name, start_time
+        time_taken = Time.now.utc - start_time.utc
+        avarage_time category, metric_name, time_taken
+      end
+
+      def avarage_time category, metric_name, elapsed_time
+        m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:counter_time]
+        m.value = elapsed_time.round #seconds
+        Stackify::Metrics.metrics_client.queue_metric m
+      end
+
+      def count_and_time category, metric_name, start_time, advanced_settings = nil
+        counter_m = Metric.new category, metric_name, Stackify::Metrics::METRIC_TYPES[:counter]
+        counter_m.value = 1
+        counter_m.settings = advanced_settings
+        time_m = Metric.new category, metric_name + ' Time', Stackify::Metrics::METRIC_TYPES[:counter_time]
+        time_m.value = (Time.now.utc - start_time.utc).round
+        time_m.settings = advanced_settings
+        Stackify::Metrics.metrics_client.queue_metric counter_m
+        Stackify::Metrics.metrics_client.queue_metric time_m
+      end
+    end
+  end
+end

data/lib/stackify/metrics/metrics_client.rb

@@ -0,0 +1,238 @@
+module Stackify::Metrics
+  class MetricsClient
+
+    attr_reader :metrics_queue
+
+    def initialize
+      @metrics_queue = MetricsQueue.new
+      @last_aggregates = {}
+      @metric_settings = {}
+      @aggregate_metrics = {}
+      @monitor_ids = {}
+      @metrics_sender = MetricsSender.new
+    end
+
+    def start
+      if Stackify::Utils.is_mode_on? Stackify::MODES[:metrics]
+        worker = Stackify::Worker.new 'Metrics client - processing of metrics'
+        Stackify.internal_log :debug, 'Metrics client: processing of metrics is started'
+        task = submit_metrics_task
+        worker.async_perform 5, task
+      else
+        Stackify.internal_log :warn, '[MetricClient]: Processing of metrics is disabled at configuration!'
+      end
+    end
+
+    def get_latest category, metric_name
+      Stackify::Utils.do_only_if_authorized_and_mode_is_on Stackify::MODES[:metrics] do
+        l = @last_aggregates.select { |_key, aggr| aggr.category.eql?(category) && aggr.name.eql?(metric_name) }
+        LatestAggregate.new l.values.first
+      end
+    end
+
+    def get_latest_all_metrics
+      Stackify::Utils.do_only_if_authorized_and_mode_is_on Stackify::MODES[:metrics] do
+        all_latest = []
+        @last_aggregates.each_pair do |_key, aggr|
+          all_latest << Stackify::Metrics::LatestAggregate.new(aggr)
+        end
+        all_latest
+      end
+    end
+
+    def queue_metric metric
+      if Stackify.working?
+        if Stackify::Utils.is_mode_on? Stackify::MODES[:metrics]
+          @metrics_queue.add_metric metric
+        else
+          Stackify.internal_log :warn, '[MetricClient]: Adding of metrics is impossible because they are disabled by configuration'
+        end
+      else
+        Stackify.internal_log :warn, '[MetricClient]: Adding of metrics is impossible - Stackify is terminating or terminated work.'
+      end
+    end
+    
+    private
+    
+    def start_upload_metrics
+      all_is_ok = false
+      current_time = Stackify::Utils.rounded_current_time
+      purge_older_than = current_time - 10.minutes
+      #read everything up to the start of the current minute
+      read_queued_metrics_batch current_time
+      handle_zero_reports current_time
+      get_for_recent = @aggregate_metrics.select do |_k, v|
+        v.occurred_utc < current_time && v.occurred_utc > current_time - 5.minutes
+      end
+      set_latest_aggregates get_for_recent
+      selected_aggr_metrics = @aggregate_metrics.select { |_key, aggr| aggr.occurred_utc < current_time }
+      first_50_metrics = Hash[selected_aggr_metrics.to_a.take 50]
+      if first_50_metrics.length > 0
+        #only getting metrics less than 10 minutes old to drop old data in case we get backed up
+        #they are removed from the @aggregated_metrics in the upload function upon success
+        all_success = upload_aggregates(first_50_metrics.select { |_key, aggr| aggr.occurred_utc > current_time - 10.minutes })
+      end
+      @aggregate_metrics.delete_if { |_key, aggr| aggr.occurred_utc < purge_older_than }
+    end
+
+    def read_queued_metrics_batch chosen_time
+      batches = {}
+
+      while  @metrics_queue.size > 0 do
+        metric = @metrics_queue.pop
+        metric.calc_and_set_aggregate_key
+        unless batches.has_key? metric.aggregate_key
+          name_key = metric.calc_name_key
+          if metric.is_increment && @last_aggregates.has_key?(name_key)
+            #if wanting to do increments we need to grab the last value so we know what to increment
+            metric.value = @last_aggregates[name_key].value
+          end
+          batches[metric.aggregate_key] = MetricAggregate.new metric
+          #if it is nil don't do anything
+          #we are doing it where the aggregates are created so we don't do it one very single metric,
+          #just once per batch to optimize performance
+
+          @metric_settings[name_key] = metric.settings if metric.settings != nil
+        end
+        batches[metric.aggregate_key].count += 1
+        if metric.is_increment
+          #add or subtract
+          batches[metric.aggregate_key].value += metric.value
+        elsif metric.metric_type == Stackify::Metrics::METRIC_TYPES[:metric_last]
+          #should end up the last value
+          batches[metric.aggregate_key].value = metric.value
+        else
+          batches[metric.aggregate_key].value += metric.value
+        end
+        #we don't need anything more this recent so bail
+        break if metric.occurred > chosen_time
+      end
+      batches.each_pair do |_key, aggregated_metric|
+        aggregate aggregated_metric
+      end
+    end
+
+    def aggregate am
+      agg_key = am.aggregate_key
+      if @aggregate_metrics.has_key? agg_key
+        agg = @aggregate_metrics[agg_key]
+      else
+        if @aggregate_metrics.length > 1000
+          str = 'No longer aggregating new metrics because more than 1000 are queued'
+          Stackify.internal_log :warn, str
+          return
+        end
+        Stackify.internal_log :debug, 'Creating aggregate for ' + agg_key
+        @aggregate_metrics[agg_key] = am
+        agg = Stackify::Metrics::Metric.new am.category, am.name, am.metric_type
+        agg = MetricAggregate.new agg
+        agg.occurred_utc = am.occurred_utc
+      end
+
+      if am.metric_type == Stackify::Metrics::METRIC_TYPES[:metric_last]
+        agg.count = 1
+        agg.value = am.value
+      else
+        agg.count += am.count
+        agg.value += am.value
+      end
+      @aggregate_metrics[agg_key]= agg
+    end
+
+    def submit_metrics_task
+      Stackify::ScheduleTask.new do 
+        start_upload_metrics
+      end
+    end
+
+    def handle_zero_reports current_time
+      @last_aggregates.each_pair do |_key, aggregate|
+        if @metric_settings.has_key? aggregate.name_key
+          settings = @metric_settings[aggregate.name_key]
+          if settings.nil?
+            @metric_settings.delete[aggregate.name_key]
+            next
+          end
+          #agg = MetricAggregate.new aggregate.category, aggregate.name, aggregate.metric_type
+          agg = aggregate
+          agg.occurred_utc = current_time
+          case aggregate.metric_type
+          when Stackify::Metrics::METRIC_TYPES[:metric_last]
+            setting.autoreport_last_value_if_nothing_reported = false #do not allow this
+          when Stackify::Metrics::METRIC_TYPES[:counter]
+            setting.autoreport_last_value_if_nothing_reported = false #do not allow this
+          end
+          if settings.autoreport_zero_if_nothing_reported
+            agg.count = 1
+            agg.value = 0
+          elsif settings.autoreport_last_value_if_nothing_reported
+            agg.count = aggregate.value.to_i
+            agg.value = aggregate.value
+          else
+            next
+          end
+          agg_key = agg.aggregate_key
+          unless @aggregate_metrics.has_key? agg_key
+            agg.name_key = aggregate.name_key
+            Stackify.internal_log :debug, 'Creating 0 default value for ' + agg_key
+            @aggregate_metrics[agg_key] = agg
+          end
+        end
+      end
+    end
+
+    def set_latest_aggregates aggregates
+      aggregates.each_pair do |_key, aggr|
+        if @last_aggregates.has_key? aggr.name_key
+          curr_aggr = @last_aggregates[aggr.name_key]
+          @last_aggregates[aggr.name_key] = aggr if aggr.occurred_utc > curr_aggr.occurred_utc
+        else
+          @last_aggregates[aggr.name_key] = aggr
+        end
+      end
+    end
+
+    def upload_aggregates aggr_metrics
+      s = ''
+      aggr_metrics.each_pair do |_k, m|
+        s = s + m.inspect.to_s + "\n --------------------------- \n"
+      end
+      Stackify.internal_log :debug, "Uploading Aggregate Metrics at #{ Time.now }: \n" + s
+      all_success = true
+      aggr_metrics.each_pair do |_key, metric|
+        if @monitor_ids.has_key? metric.name_key
+          mon_info = @monitor_ids[metric.name_key]
+        else
+          req = @metrics_sender.monitor_info metric
+          if req.try(:code) == '200'
+            mon_info = JSON.parse req.body
+            if !mon_info.nil? && !mon_info['MonitorID'].nil? && mon_info['MonitorID'] > 0
+              @monitor_ids[metric.name_key] = mon_info
+            elsif !mon_info.nil? && mon_info['MonitorID'].nil?
+              Stackify.internal_log :warn, 'Unable to get metric info for ' + metric.name_key + ' MonitorID is nil'
+              @monitor_ids[metric.name_key] = mon_info
+            end
+          else
+            Stackify.internal_log :error, 'Unable to get metric info for ' + metric.name_key
+            mon_info = nil
+            all_success = false
+          end
+        end
+
+        if mon_info.nil? || mon_info['MonitorID'].nil?
+          Stackify.internal_log :warn, 'Metric info missing for ' + metric.name_key
+          metric.monitor_id = nil
+          all_success = false
+        else
+          metric.monitor_id = mon_info['MonitorID']
+        end
+
+        #get identified once
+        aggr_metrics_for_upload = aggr_metrics.select { |_key, aggr_metric| !aggr_metric.monitor_id.nil? }
+        response = @metrics_sender.upload_metrics aggr_metrics_for_upload
+        Stackify.internal_log :info, 'Metrics are uploaded successfully' if response.try(:code) == '200'
+        all_success
+      end
+    end
+  end
+end