clear_skies 0.4.1 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71658311ffdc9ed619d871de1c83e0536f973af9
-  data.tar.gz: a25bb06314702201194df39f39eb07456753aff6
+  metadata.gz: 4663b9f343938c385dbea126a760ead8b1680241
+  data.tar.gz: 83704537b0a74a203504639974d8c77ff36df7fe
 SHA512:
-  metadata.gz: ef27ddde07cc95c78d34dec964bf3251ee129d91081f7a0d2d0d1775b8a774f54a106c66dd6541dcffe69741812942ced6d56d93cd7e55dea62129f3b92eeb83
-  data.tar.gz: 129defa2139170c0a31133dedc835712ef912c3055070d2c2dfcd182b97760f09ae250329d1cff6677f020f317cbb08b7aa466bd6d4b81f4fd03e6844e0dab60
+  metadata.gz: 37c48599ec83e12fafca817bb478ffa10f34034a7c51e97516949c085edd8e6ac2d2f6c7db6555555881e1ee88dfe383d53b508adf47e235f4cbbec810e8e6ed
+  data.tar.gz: dcf63b1909ad72ed5e4c595b6ed9f490e61449f9f6edbd715d394d1ccbad79153846a18a4192a0e9e9995040ca8c8e09f8bc28597d2119134270e2cca3b696fa

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    clear_skies (0.4.1)
+    clear_skies (0.5.4)
       awesome_print
       aws-sdk
       elasticsearch
@@ -74,7 +74,7 @@ GEM
       multipart-post (>= 1.2, < 3)
     globalid (0.4.0)
       activesupport (>= 4.2.0)
-    greek_fire (0.4.1)
+    greek_fire (0.5.0)
       rails (>= 4.2.0)
     i18n (0.8.6)
     jmespath (1.3.1)
@@ -87,7 +87,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
     mini_portile2 (2.2.0)
-    minitest (5.10.2)
+    minitest (5.10.3)
     multi_json (1.12.1)
     multipart-post (2.0.0)
     nio4r (2.1.0)

data/lib/clear_skies.rb

@@ -13,6 +13,7 @@ require "clear_skies/aws/cloud_watch/billing"
 require "clear_skies/aws/cloud_watch/rds_gauge"
 require "clear_skies/aws/cloud_watch/elb_gauge"
 require "clear_skies/aws/cloud_watch/elastic_beanstalk_gauge"
+require "clear_skies/aws/cloud_watch/redshift_gauge"
 require "clear_skies/aws/cloud_watch/request_counter"
 require "clear_skies/redis/report"
 require "clear_skies/elasticsearch/report"

data/lib/clear_skies/aws/cloud_watch/billing.rb

@@ -10,6 +10,10 @@ module ClearSkies
         def self.cloudwatch_client
           @client ||= Aws::CloudWatch::Client.new(region: "us-east-1")
         end
+
+        def metrics
+          Rails.cache.fetch(self.to_s, expires_in: 1.hour) { super }
+        end
       end
     end
   end

data/lib/clear_skies/aws/cloud_watch/elastic_beanstalk_gauge.rb

@@ -5,8 +5,8 @@ module ClearSkies
         def self.beanstalk_client
           @beanstalk_client ||= ::Aws::ElasticBeanstalk::Client.new
         end
-        def initialize(metric_name, dimension, statistics, description: nil, &block)
-          super("AWS/ElasticBeanstalk", metric_name, dimension, statistics, description: description, &block)
+        def initialize(metric_name, dimension, statistics, description: nil, aws_parameters:nil, &block)
+          super("AWS/ElasticBeanstalk", metric_name, dimension, statistics, description: description, aws_parameters: aws_parameters, &block)
         end
 
         def application_name(environment_name)

data/lib/clear_skies/aws/cloud_watch/elb_gauge.rb

@@ -6,8 +6,8 @@ module ClearSkies
         def self.elb_client
           @elb_client ||= Aws::ElasticLoadBalancing::Client.new
         end
-        def initialize(metric_name, dimension, statistics, description: nil, &block)
-          super("AWS/ELB", metric_name, dimension, statistics, description: description, &block)
+        def initialize(metric_name, dimension, statistics, description: nil, aws_parameters:nil, &block)
+          super("AWS/ELB", metric_name, dimension, statistics, description: description, aws_parameters: aws_parameters, &block)
         end
 
         def tags(load_balancer_name)

data/lib/clear_skies/aws/cloud_watch/gauge.rb

@@ -46,10 +46,14 @@ module ClearSkies
 
             next unless @block.call(labels) if @block
 
+            start_time = Time.now.advance(@aws_parameters[:start_time] || {minutes: -6})
+            end_time   = Time.now.advance(@aws_parameters[:end_time]   || {minutes: -5})
+            period     = @aws_parameters[:period] || end_time - start_time
+
             stats = metric.get_statistics(
-                start_time: Time.now.advance(@aws_parameters[:start_time] || {minutes: -6}),
-                end_time: Time.now.advance(@aws_parameters[:end_time] || {minutes: -5}),
-                period: @aws_parameters[:period] || 1,
+                start_time: start_time,
+                end_time: end_time,
+                period: period,
                 statistics: @statistics,
                 extended_statistics: @extended_statistics,
                 dimensions: metric.dimensions

data/lib/clear_skies/aws/cloud_watch/rds_gauge.rb

@@ -3,8 +3,8 @@ module ClearSkies
 
     module CloudWatch
       class RDSGauge < ClearSkies::AWS::CloudWatch::Gauge
-        def initialize(metric_name, dimension, statistics, description: nil, &block)
-          super("AWS/RDS", metric_name, dimension, statistics, description: description, &block)
+        def initialize(metric_name, dimension, statistics, description: nil, aws_parameters:nil, &block)
+          super("AWS/RDS", metric_name, dimension, statistics, description: description, aws_parameters: aws_parameters, &block)
         end
 
         def tags(db)

data/lib/clear_skies/aws/cloud_watch/redshift_gauge.rb

@@ -0,0 +1,40 @@
+module ClearSkies
+  module AWS
+    module CloudWatch
+      class RedshiftGauge < ClearSkies::AWS::CloudWatch::Gauge
+        def initialize(metric_name, dimension, statistics, description: nil, aws_parameters:nil, &block)
+          super("AWS/Redshift", metric_name, dimension, statistics, description: description, aws_parameters: aws_parameters, &block)
+        end
+
+        def extra_labels(cluster_id)
+          labels = {}
+
+          cluster = Aws::Redshift::Client.new.describe_clusters({
+                                                                    cluster_identifier: cluster_id
+                                                                }).clusters[0]
+
+          vpc_id = cluster.vpc_id
+          labels["vpc_id"] = vpc_id if vpc_id.present?
+
+          cluster.tags.each do |tag|
+            labels[tag.key.downcase] = tag.value
+          end
+          labels
+        rescue Aws::Redshift::Errors::ClusterNotFound
+          return labels
+        end
+
+        def labels_from_metric(metric)
+          labels = super(metric)
+
+          if labels.has_key?("cluster_identifier")
+            labels.merge!(Rails.cache.fetch("#{labels["cluster_identifier"]}_extra_labels_", expires_in: 1.hour, race_condition_ttl: 60.seconds) do
+              extra_labels(labels["cluster_identifier"])
+            end)
+          end
+          labels
+        end
+      end
+    end
+  end
+end

data/lib/clear_skies/elasticsearch/report.rb

@@ -1,4 +1,5 @@
 require 'elasticsearch'
+require 'uri'
 
 module ClearSkies
   module Elasticsearch
@@ -12,65 +13,157 @@ module ClearSkies
         @reports ||= []
       end
 
-      attr_reader :url, :extra_labels, :metrics
+
+      def self.flatten_hash(hash, prefix="elasticsearch")
+        ret = HashWithIndifferentAccess.new
+        hash.each do |k, value|
+          name = "#{prefix}_#{k}"
+          if value.is_a?(Hash)
+            ret.merge!(flatten_hash(value, name))
+          elsif(value.is_a?(Numeric))
+            ret[name] = value
+          end
+        end
+        return ret
+      end
+
+      attr_reader :url, :extra_labels
       def initialize(url, extra_labels)
         @url = url
-        @extra_labels = extra_labels
-        GreekFire::Measure.before_metrics { refresh }
+
+        @extra_labels = {host: URI(url).host}.merge extra_labels
       end
 
-      def elasticsearch_metrics(client, index)
-        metrics = OpenStruct.new
-        metrics.index = index
-        stats = client.indices.stats["indices"][index]["total"]
-        metrics.docs_count = stats["docs"]["count"]
-        metrics.docs_deleted = stats["docs"]["deleted"]
-        metrics
+      def metrics
+        client = ::Elasticsearch::Client.new(hosts: url)
+        stats = client.cluster.stats
+        ClearSkies::Elasticsearch::Report.flatten_hash(stats)
       end
 
-      def refresh
+      def indicies
         client = ::Elasticsearch::Client.new(hosts: @url)
-        aliases = client.indices.get_aliases
+        client.indices.get_aliases
+      end
 
-        @metrics = aliases.map do |index, _aliases|
-          elasticsearch_metrics(client, index)
+      def report_indicies
+        indicies.map do |index|
+          ReportIndex.new(self, index)
         end
       end
 
-    end
+      def nodes
+        client = ::Elasticsearch::Client.new(hosts: @url)
+        client.nodes.stats["nodes"]
+      end
 
-    class Measure < GreekFire::Measure
-      def initialize(name, &block)
-        super(name) do |label|
-          block.call(label.delete(:metric))
+      def report_nodes
+        nodes.map do |node_name, node_props|
+          ReportNode.new(self, node_name, node_props)
         end
       end
+    end
 
+    class ReportIndex
+      attr_reader :report, :index
 
-      def labels
-        ClearSkies::Elasticsearch::Report.reports.map do |report|
-          report.metrics.map do |metric|
-            report.extra_labels.merge({index: metric.index, metric: metric })
-          end
-        end.flatten
+      def initialize(report, index)
+        @report = report
+        @index = index
+        @extra_labels = { index: index_alias }
+      end
+
+      def index_alias
+        @index[1]["aliases"].keys&.first || @index[0]
+      end
+
+      def index_name
+        @index[0]
+      end
+
+      def extra_labels
+        @report.extra_labels.merge(@extra_labels)
+      end
+
+
+      def metrics
+        client = ::Elasticsearch::Client.new(hosts: report.url)
+        stats = client.indices.stats["indices"][index_name]["total"]
+        ClearSkies::Elasticsearch::Report.flatten_hash(stats)
       end
     end
 
-    class Gauge < ClearSkies::Elasticsearch::Measure
-      def to_partial_path
-        GreekFire::Gauge._to_partial_path
+    class ReportNode
+      attr_reader :report, :extra_labels
+
+      def initialize(report, node_name, node_props)
+        @report = report
+        @node_props = node_props
+        @extra_labels = { node_name: node_name }
+      end
+
+      def extra_labels
+        @report.extra_labels.merge(@extra_labels)
+      end
+
+      def metrics
+        ClearSkies::Elasticsearch::Report.flatten_hash(@node_props, "elasticsearch_node")
       end
+
     end
-    class Counter < ClearSkies::Elasticsearch::Measure
-      def to_partial_path
-        GreekFire::Counter._to_partial_path
+
+
+    class MeasureSet < GreekFire::MeasureSet
+      def items
+        metrics = []
+
+        ### Cluster Metrics ##
+        labels = ClearSkies::Elasticsearch::Report.reports.map do | report|
+          GreekFire::SmartLabel.new(report.metrics, report.extra_labels)
+        end
+
+        metric_names = labels.map {|i| i.value.keys}.flatten.uniq
+
+        metric_names.each do |metric|
+          metrics << GreekFire::Gauge.new(metric, labels: labels) do |label|
+            label.value[metric]
+          end
+        end
+
+        ### Index Metrics
+        labels = ClearSkies::Elasticsearch::Report.reports.map do | report|
+          report.report_indicies.map do |report_index|
+            GreekFire::SmartLabel.new(report_index.metrics, report_index.extra_labels)
+          end
+        end.flatten
+
+        metric_names = labels.map {|i| i.value.keys}.flatten.uniq
+
+        metric_names.each do |metric|
+          metrics << GreekFire::Gauge.new(metric, labels: labels) do |label|
+            label.value[metric]
+          end
+        end
+
+        ### Node Metrics
+        labels = ClearSkies::Elasticsearch::Report.reports.map do | report|
+          report.report_nodes.map do |report_node|
+            GreekFire::SmartLabel.new(report_node.metrics, report_node.extra_labels)
+          end
+        end.flatten
+
+        metric_names = labels.map {|i| i.value.keys}.flatten.uniq
+
+        metric_names.each do |metric|
+          metrics << GreekFire::Gauge.new(metric, labels: labels) do |label|
+            label.value[metric]
+          end
+        end
+
+        metrics
       end
     end
+
   end
 end
 
-# GreekFire::Metric.register(ClearSkies::Elasticsearch::Gauge.new("elasticsearch_docs_count")                     { |metrics| metrics.docs_count })
-# GreekFire::Metric.register(ClearSkies::Elasticsearch::Gauge.new("elasticsearch_docs_deleted")                   { |metrics| metrics.docs_deleted })
-
-
-#For a <DIM>, grab N metrics.
+GreekFire::Metric.register ClearSkies::Elasticsearch::MeasureSet.new

data/lib/clear_skies/redis/report.rb

@@ -29,16 +29,17 @@ module ClearSkies
           ReportDimension.new(self, dimension)
         end
       end
-
-      def items
-        report_dimensions.map
-      end
     end
 
     class MeasureSet < GreekFire::MeasureSet
       def items
-        report_dimensions = ClearSkies::Redis::Report.reports.map(&:report_dimensions).flatten
-        return [] unless report_dimensions.length > 0
+        labels = ClearSkies::Redis::Report.reports.map do | report|
+          report.report_dimensions.map do |report_dimension|
+            GreekFire::SmartLabel.new(report_dimension.metrics, report_dimension.extra_labels)
+          end
+        end.flatten
+
+        return [] unless labels.length > 0
 
         [
             "keys",
@@ -52,7 +53,9 @@ module ClearSkies
             "rdb_last_bgsave_time_sec",
             "total_commands_processed"
         ].map do |metric_name|
-          ClearSkies::Redis::Gauge.new(report_dimensions,  "redis", metric_name)
+          GreekFire::Gauge.new("redis_#{metric_name}", labels: labels) do |label|
+            label.value[metric_name.to_sym]
+          end
         end
       end
     end
@@ -63,53 +66,31 @@ module ClearSkies
       def initialize(report, dimension)
         @report = report
         @dimension = dimension
+        @extra_labels = { db: dimension }
+      end
+
+      def extra_labels
+        @report.extra_labels.merge(@extra_labels)
       end
 
       def metrics
         redis = ::Redis.new(host: report.host, port: report.port, db: @dimension)
         redis_info = redis.info
 
-        metrics = OpenStruct.new
-        metrics.keys = redis.dbsize
-        metrics.last_save = Time.now.to_i - redis.lastsave
-        metrics.uptime = redis_info["uptime_in_seconds"].to_f
-        metrics.connected_clients =  redis_info["connected_clients"].to_i
-        metrics.blocked_clients =  redis_info["blocked_clients"].to_i
-        metrics.used_memory =  redis_info["used_memory"].to_f
-        metrics.mem_fragmentation_ratio =  redis_info["mem_fragmentation_ratio"].to_f
-        metrics.rdb_changes_since_last_save =  redis_info["rdb_changes_since_last_save"].to_f
-        metrics.rdb_last_bgsave_time_szec =  redis_info["rdb_last_bgsave_time_sec"].to_f
-        metrics.total_commands_processed =  redis_info["total_commands_processed"].to_f
+        metrics = HashWithIndifferentAccess.new
+        metrics[:keys ]= redis.dbsize
+        metrics[:last_save ]= Time.now.to_i - redis.lastsave
+        metrics[:uptime ]= redis_info["uptime_in_seconds"].to_f
+        metrics[:connected_clients ]=  redis_info["connected_clients"].to_i
+        metrics[:blocked_clients ]=  redis_info["blocked_clients"].to_i
+        metrics[:used_memory ]=  redis_info["used_memory"].to_f
+        metrics[:mem_fragmentation_ratio ]=  redis_info["mem_fragmentation_ratio"].to_f
+        metrics[:rdb_changes_since_last_save ]=  redis_info["rdb_changes_since_last_save"].to_f
+        metrics[:rdb_last_bgsave_time_szec ]=  redis_info["rdb_last_bgsave_time_sec"].to_f
+        metrics[:total_commands_processed ]=  redis_info["total_commands_processed"].to_f
         metrics
       end
     end
-
-    class Measure < GreekFire::Measure
-      def initialize(report_dimensions, prefix, name)
-        @report_dimensions = report_dimensions
-        super("#{prefix}_#{name}") do |label|
-          label.delete(:metric).metrics.to_h[name.to_sym]
-        end
-      end
-
-
-      def labels
-        @report_dimensions.map do |report_dimension|
-          report_dimension.report.extra_labels.merge({metric: report_dimension })
-        end
-      end
-    end
-
-    class Gauge < ClearSkies::Redis::Measure
-      def to_partial_path
-        GreekFire::Gauge._to_partial_path
-      end
-    end
-    class Counter < ClearSkies::Redis::Measure
-      def to_partial_path
-        GreekFire::Counter._to_partial_path
-      end
-    end
   end
 end
 

data/lib/clear_skies/version.rb

@@ -1,3 +1,3 @@
 module ClearSkies
-  VERSION = "0.4.1"
+  VERSION = "0.5.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clear_skies
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.4
 platform: ruby
 authors:
 - Chris Constantine
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-17 00:00:00.000000000 Z
+date: 2017-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puma
@@ -150,6 +150,7 @@ files:
 - lib/clear_skies/aws/cloud_watch/elb_gauge.rb
 - lib/clear_skies/aws/cloud_watch/gauge.rb
 - lib/clear_skies/aws/cloud_watch/rds_gauge.rb
+- lib/clear_skies/aws/cloud_watch/redshift_gauge.rb
 - lib/clear_skies/aws/cloud_watch/request_counter.rb
 - lib/clear_skies/aws/reservation_utilization.rb
 - lib/clear_skies/elasticsearch/report.rb