clear_skies 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b28817b1d9a2d115177165c9b1a2ab8d44aeb43
-  data.tar.gz: 2033b9f62ed98513dc3eb01c3a3ab88d682d2860
+  metadata.gz: 82eb78bcb0b8d0e46d77a658b8a46390441f6a61
+  data.tar.gz: 96bd95ac7fe050aab5c42705751651d465cc00fc
 SHA512:
-  metadata.gz: 86fbbf4d79173b447bd66d5539263ede10256bf23c129397b032ebed7e035560ee899f3b067165327ac149f44636f234cf0aa5519479bf7d6132c44774685fe4
-  data.tar.gz: 4333fa6071ffeb618af49418fa01e644cf4729b4bf7ec996c45c5b1bdaf8cd99d81739ecc0f51ff7166b98b1792156b3ef7d6e257e3dac0a7214943ff21b143e
+  metadata.gz: ea88cec568d392ecdcaa6e9907c43d6d06558348234d83e5cfc4c6258377b986c0f4e0f38a778bbc9d9e3585f4543cff62c638207061debd3b887b741df4a1cd
+  data.tar.gz: adac7ec7203afb00fd939cabf39ea7b3e22d146946667372475506f3238727bb030e86345ae45e8ae7adaec1a9c8321c06b84c927016992183890ec9fd24b8e6

data/.gitignore

@@ -1,7 +1,6 @@
 /.bundle/
 /.yardoc
 .idea
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

data/Gemfile.lock

@@ -0,0 +1,149 @@
+PATH
+  remote: .
+  specs:
+    clear_skies (0.4.0)
+      awesome_print
+      aws-sdk
+      elasticsearch
+      greek_fire (>= 0.3.0)
+      puma
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (5.1.2)
+      actionpack (= 5.1.2)
+      nio4r (~> 2.0)
+      websocket-driver (~> 0.6.1)
+    actionmailer (5.1.2)
+      actionpack (= 5.1.2)
+      actionview (= 5.1.2)
+      activejob (= 5.1.2)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.1.2)
+      actionview (= 5.1.2)
+      activesupport (= 5.1.2)
+      rack (~> 2.0)
+      rack-test (~> 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.1.2)
+      activesupport (= 5.1.2)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.1.2)
+      activesupport (= 5.1.2)
+      globalid (>= 0.3.6)
+    activemodel (5.1.2)
+      activesupport (= 5.1.2)
+    activerecord (5.1.2)
+      activemodel (= 5.1.2)
+      activesupport (= 5.1.2)
+      arel (~> 8.0)
+    activesupport (5.1.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (8.0.0)
+    awesome_print (1.8.0)
+    aws-sdk (2.10.10)
+      aws-sdk-resources (= 2.10.10)
+    aws-sdk-core (2.10.10)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.10.10)
+      aws-sdk-core (= 2.10.10)
+    aws-sigv4 (1.0.0)
+    builder (3.2.3)
+    concurrent-ruby (1.0.5)
+    elasticsearch (5.0.4)
+      elasticsearch-api (= 5.0.4)
+      elasticsearch-transport (= 5.0.4)
+    elasticsearch-api (5.0.4)
+      multi_json
+    elasticsearch-transport (5.0.4)
+      faraday
+      multi_json
+    erubi (1.6.1)
+    faraday (0.12.1)
+      multipart-post (>= 1.2, < 3)
+    globalid (0.4.0)
+      activesupport (>= 4.2.0)
+    greek_fire (0.4.1)
+      rails (>= 4.2.0)
+    i18n (0.8.6)
+    jmespath (1.3.1)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    mail (2.6.6)
+      mime-types (>= 1.16, < 4)
+    method_source (0.8.2)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.2.0)
+    minitest (5.10.2)
+    multi_json (1.12.1)
+    multipart-post (2.0.0)
+    nio4r (2.1.0)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
+    puma (3.9.1)
+    rack (2.0.3)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails (5.1.2)
+      actioncable (= 5.1.2)
+      actionmailer (= 5.1.2)
+      actionpack (= 5.1.2)
+      actionview (= 5.1.2)
+      activejob (= 5.1.2)
+      activemodel (= 5.1.2)
+      activerecord (= 5.1.2)
+      activesupport (= 5.1.2)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 5.1.2)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    railties (5.1.2)
+      actionpack (= 5.1.2)
+      activesupport (= 5.1.2)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.5.0)
+    redis (3.3.3)
+    sprockets (3.7.1)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.0)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    thor (0.19.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    websocket-driver (0.6.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.14)
+  clear_skies!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.15.1

data/clear_skies.gemspec

@@ -24,7 +24,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "puma"
   spec.add_dependency "aws-sdk"
   spec.add_dependency "redis"
+  spec.add_dependency "elasticsearch"
   spec.add_dependency "greek_fire", ">= 0.3.0"
+
   spec.add_dependency "awesome_print"
 
   spec.add_development_dependency "bundler", "~> 1.14"

data/lib/clear_skies/aws/cloud_watch/billing.rb

@@ -0,0 +1,16 @@
+module ClearSkies
+  module AWS
+    module CloudWatch
+      class Billing < ClearSkies::AWS::CloudWatch::Gauge
+        def initialize(dimension, statistics, description: nil, &block)
+          super("AWS/Billing", "EstimatedCharges", dimension, statistics, description: description, aws_parameters: {start_time: {days: -1}, end_time: {minutes: -5}, period: 60*60*24}, &block)
+        end
+
+
+        def self.cloudwatch_client
+          @client ||= Aws::CloudWatch::Client.new(region: "us-east-1")
+        end
+      end
+    end
+  end
+end

data/lib/clear_skies/aws/cloud_watch/elastic_beanstalk_gauge.rb

@@ -0,0 +1,47 @@
+module ClearSkies
+  module AWS
+    module CloudWatch
+      class ElasticBeanstalkGauge < ClearSkies::AWS::CloudWatch::Gauge
+        def self.beanstalk_client
+          @beanstalk_client ||= ::Aws::ElasticBeanstalk::Client.new
+        end
+        def initialize(metric_name, dimension, statistics, description: nil, &block)
+          super("AWS/ElasticBeanstalk", metric_name, dimension, statistics, description: description, &block)
+        end
+
+        def application_name(environment_name)
+          ElasticBeanstalkGauge.beanstalk_client.describe_environments({environment_names: [environment_name] }).environments.first&.application_name
+        end
+
+        def vpc_id(application_name, environment_name)
+          config = ElasticBeanstalkGauge.beanstalk_client.describe_configuration_settings({ application_name: application_name, environment_name: environment_name }).
+              configuration_settings.find { |config| config.application_name == application_name && config.environment_name == environment_name}
+          option =  config.option_settings.find { |option| option.namespace == "aws:ec2:vpc" && option.option_name == "VPCId"}
+          option.value if option
+        end
+
+
+        def labels_from_metric(metric)
+          labels = super(metric)
+
+          if labels.has_key?( "environment_name") && !(Rails.cache.fetch("#{labels["environment_name"]}_skip"))
+            application_name = Rails.cache.fetch("#{labels["environment_name"]}_application_name", expires_in: 1.hour) do
+              application_name(labels["environment_name"])
+            end
+            if application_name
+              labels["application_name"] =  application_name
+
+              labels["vpc_id"] = Rails.cache.fetch("#{labels["environment_name"]}_vpc_id_") do
+                vpc_id(labels["application_name"], labels["environment_name"])
+              end
+            end
+          else
+            Rails.cache.write("#{labels["environment_name"]}_skip", true, expires_in: 1.hour, race_condition_ttl: 60.seconds)
+          end
+
+          return labels
+        end
+      end
+    end
+  end
+end

data/lib/clear_skies/aws/cloud_watch/elb_gauge.rb

@@ -0,0 +1,47 @@
+module ClearSkies
+  module AWS
+
+    module CloudWatch
+      class ELBGauge < ClearSkies::AWS::CloudWatch::Gauge
+        def self.elb_client
+          @elb_client ||= Aws::ElasticLoadBalancing::Client.new
+        end
+        def initialize(metric_name, dimension, statistics, description: nil, &block)
+          super("AWS/ELB", metric_name, dimension, statistics, description: description, &block)
+        end
+
+        def tags(load_balancer_name)
+          labels = {}
+          ELBGauge.elb_client.
+              describe_tags({load_balancer_names: [load_balancer_name]}).
+              tag_descriptions.
+              select {|doc| doc.load_balancer_name == load_balancer_name}.each do |tag_description|
+            tag_description.tags.each do |tag|
+              labels[tag.key.downcase] = tag.value
+            end
+          end
+          labels
+        end
+
+        def labels_from_metric(metric)
+          labels = super(metric)
+
+          if labels.has_key?( "load_balancer_name") && !(Rails.cache.fetch("#{labels["load_balancer_name"]}_skip"))
+
+            labels["vpc_id"] = Rails.cache.fetch("#{labels["load_balancer_name"]}_vpc_id_") do
+              ELBGauge.elb_client.describe_load_balancers(load_balancer_names: [labels["load_balancer_name"]]).load_balancer_descriptions.first.vpc_id
+            end
+
+            labels.merge!(Rails.cache.fetch("#{labels["load_balancer_name"]}_tags_", expires_in: 1.hour, race_condition_ttl: 60.seconds) do
+              tags(labels["load_balancer_name"])
+            end)
+          end
+          labels
+        rescue Aws::ElasticLoadBalancing::Errors::LoadBalancerNotFound
+          Rails.cache.write("#{labels["load_balancer_name"]}_skip", true, expires_in: 1.hour, race_condition_ttl: 60.seconds)
+          return labels
+        end
+      end
+    end
+  end
+end

data/lib/clear_skies/aws/cloud_watch/gauge.rb

@@ -0,0 +1,73 @@
+
+module ClearSkies
+  module AWS
+    module CloudWatch
+      class Gauge < GreekFire::Gauge
+        include ActiveModel::Conversion
+
+        def self.register(*args, &block)
+          GreekFire::Metric.register(self.new(*args, &block))
+        end
+
+        def initialize(namespace, metric_name, dimensions, statistics, description:nil, aws_parameters:nil, &block)
+          super("#{namespace.underscore.gsub("/", "_")}_#{metric_name.underscore}", description: description)
+          @namespace = namespace
+          @metric_name = metric_name
+          @dimensions = dimensions
+          @statistics = statistics.select { |stat| ["SampleCount", "Average", "Sum", "Minimum", "Maximum"].include?(stat.to_s) }
+          @extended_statistics = statistics - @statistics
+          @aws_parameters = aws_parameters || { }
+
+          @block = block
+        end
+
+        def self.cloudwatch_client
+          @client ||= Aws::CloudWatch::Client.new
+        end
+
+        def aws_metrics
+           Aws::CloudWatch::Resource.new(client: self.class.cloudwatch_client).metrics(
+              namespace: @namespace,
+              metric_name: @metric_name,
+              dimensions: @dimensions.map {|dimension| {name: dimension} }
+          ).select { |metrics| metrics.dimensions.count == @dimensions.count }
+        end
+
+        def labels_from_metric(metric)
+          metric.dimensions.inject(ActiveSupport::HashWithIndifferentAccess.new) do |labels, dimension|
+            labels[dimension.name.underscore] = dimension.value
+            labels
+          end
+        end
+
+        def metrics
+          aws_metrics.map do |metric|
+            labels = labels_from_metric(metric)
+
+            next unless @block.call(labels) if @block
+
+            stats = metric.get_statistics(
+                start_time: Time.now.advance(@aws_parameters[:start_time] || {minutes: -6}),
+                end_time: Time.now.advance(@aws_parameters[:end_time] || {minutes: -5}),
+                period: @aws_parameters[:period] || 1,
+                statistics: @statistics,
+                extended_statistics: @extended_statistics,
+                dimensions: metric.dimensions
+            )
+
+            stats.datapoints.map do |datapoint|
+              datapoint.to_h.select {|k, v| ![:unit, :timestamp].include?(k) }.map do |key, value|
+                if (key == :extended_statistics)
+                  value.map {|e_key, e_value| GreekFire::Metric.new(name, labels.merge({statistic: e_key}), e_value)}
+                else
+                  GreekFire::Metric.new(name, labels.merge({statistic: key}), value)
+                end
+              end
+            end
+
+          end.flatten.compact
+        end
+      end
+    end
+  end
+end

data/lib/clear_skies/aws/cloud_watch/rds_gauge.rb

@@ -0,0 +1,42 @@
+module ClearSkies
+  module AWS
+
+    module CloudWatch
+      class RDSGauge < ClearSkies::AWS::CloudWatch::Gauge
+        def initialize(metric_name, dimension, statistics, description: nil, &block)
+          super("AWS/RDS", metric_name, dimension, statistics, description: description, &block)
+        end
+
+        def tags(db)
+          labels = {}
+          db.client.list_tags_for_resource(resource_name: db.db_instance_arn).tag_list.each do |tag|
+            labels[tag.key.downcase] = tag.value
+          end
+          labels
+        end
+
+        def labels_from_metric(metric)
+          labels = super(metric)
+
+          if labels.has_key?( "db_instance_identifier") && !(Rails.cache.fetch("#{labels["db_instance_identifier"]}_skip"))
+            db = Aws::RDS::DBInstance.new(labels["db_instance_identifier"])
+
+            vpc_id = Rails.cache.fetch("#{labels["db_instance_identifier"]}_vpc_id_") do
+              db.db_subnet_group&.vpc_id
+            end
+
+            labels["vpc_id"] = vpc_id if vpc_id.present?
+
+            labels.merge!(Rails.cache.fetch("#{labels["db_instance_identifier"]}_tags_", expires_in: 1.hour, race_condition_ttl: 60.seconds) do
+              tags(db)
+            end)
+          end
+          labels
+        rescue Aws::RDS::Errors::DBInstanceNotFound
+          Rails.cache.write("#{labels["db_instance_identifier"]}_skip", true, expires_in: 1.hour, race_condition_ttl: 60.seconds)
+          return labels
+        end
+      end
+    end
+  end
+end

data/lib/clear_skies/aws/cloud_watch/request_counter.rb

@@ -0,0 +1,40 @@
+module ClearSkies
+  module AWS
+
+    module CloudWatch
+      class RequestCounter < Seahorse::Client::Handler
+        @mutex = Mutex.new
+        def self.increment
+          @mutex.synchronize do
+            @count ||= 0
+            @count += 1
+          end
+        end
+
+        def self.count
+          @count ||= 0
+          @count
+        end
+
+        def initialize(handler)
+          @handler = handler
+        end
+
+        def call(context)
+          RequestCounter.increment
+          @handler.call(context)
+        end
+      end
+
+      class RequestCounterPlugin < Seahorse::Client::Plugin
+
+        def add_handlers(handlers, config)
+          handlers.add(RequestCounter, step: :validate)
+        end
+
+      end
+    end
+  end
+end
+
+Aws::CloudWatch::Client.add_plugin(ClearSkies::AWS::CloudWatch::RequestCounterPlugin)

data/lib/clear_skies/aws/reservation_utilization.rb

@@ -0,0 +1,129 @@
+module ClearSkies
+  module AWS
+    class ReservationUtilization < GreekFire::MeasureSet
+      def items
+        client = Aws::EC2::Client.new()
+        reservations = client.describe_reserved_instances(filters: [{name: "state", values: ["active"]}]).reserved_instances.map { |x| ReservationMatcher.new(x) }
+
+        instance_counts = Hash.new { 0 }
+
+
+        instance_spawns = client.describe_instances(filters: [{name: "instance-state-name", values: ["running"]}])
+
+        instance_spawns.reservations.each do |spawn|
+          spawn.instances.each do |instance|
+            reservations.find { |reservation| reservation.match(instance) }
+
+            instance_counts[{instance_type: instance.instance_type, availability_zone: instance.placement.availability_zone, tenancy: instance.placement.tenancy}] += 1
+          end
+        end
+
+        [
+            ReservationExpirationGauge.new(reservations),
+            ReservationPurchasedGauge.new(reservations),
+            ReservationUsageGauge.new(reservations),
+            InstancesGauge.new(instance_counts)
+        ]
+      end
+    end
+
+    class ReservationMatcher
+      attr_reader :reservation, :match_count, :instance_count
+
+      def initialize(reservation)
+        @reservation = reservation
+        @instance_count = reservation.instance_count
+
+        @match_count = 0
+      end
+
+      def expires_in
+        @reservation.end - Time.now
+      end
+
+      def match(instance)
+        return false if match_count >= instance_count
+        return false unless @reservation.instance_type == instance.instance_type
+        return false unless @reservation.scope == "Region" || @reservation.availability_zone == instance.placement.availability_zone
+        return false unless @reservation.instance_tenancy == instance.placement.tenancy
+
+        @match_count += 1
+        return true
+      end
+
+      def labels
+        {
+            reserved_instances_id: @reservation.reserved_instances_id,
+            instance_type: @reservation.instance_type,
+            availability_zone: @reservation.availability_zone || "Region",
+            tenancy: @reservation.instance_tenancy,
+        }
+      end
+    end
+
+    class InstancesGauge < GreekFire::Gauge
+      def initialize(instance_counts)
+        super("aws_ec2_instances", description: "Number of instances running by type") do |labels|
+          labels.delete(:count)
+        end
+
+        @instance_counts = instance_counts
+      end
+
+      def labels
+        @instance_counts.map do |labels, count|
+          labels.merge(count: count)
+        end
+      end
+    end
+
+    class ReservationPurchasedGauge < GreekFire::Gauge
+      def initialize(reservations)
+        super("aws_ec2_reservation_purchases", description: "Number of instance reservations purchased") do |labels|
+          labels.delete(:reservation).instance_count
+        end
+
+        @reservations = reservations
+      end
+
+      def labels
+        @reservations.map do |reservation|
+          reservation.labels.merge(reservation: reservation)
+        end
+      end
+    end
+
+    class ReservationUsageGauge < GreekFire::Gauge
+      def initialize(reservations)
+        super("aws_ec2_reservation_usage", description: "Number of instance reservations in use") do |labels|
+          labels.delete(:reservation).match_count
+        end
+
+        @reservations = reservations
+      end
+
+      def labels
+        @reservations.map do |reservation|
+          reservation.labels.merge(reservation: reservation)
+        end
+      end
+    end
+
+    class ReservationExpirationGauge < GreekFire::Gauge
+      def initialize(reservations)
+        super("aws_ec2_reservation_expire_in", description: "seconds until instance reservation expires") do |labels|
+          labels.delete(:reservation).expires_in
+        end
+
+        @reservations = reservations
+      end
+
+      def labels
+        @reservations.map do |reservation|
+          reservation.labels.merge(reservation: reservation)
+        end
+      end
+    end
+
+  end
+end

data/lib/clear_skies/elasticsearch/report.rb

@@ -0,0 +1,76 @@
+require 'elasticsearch'
+
+module ClearSkies
+  module Elasticsearch
+    class Report
+
+      def self.register(url, extra_labels=nil)
+        reports << ClearSkies::Elasticsearch::Report.new(url, extra_labels || Hash.new)
+      end
+
+      def self.reports
+        @reports ||= []
+      end
+
+      attr_reader :url, :extra_labels, :metrics
+      def initialize(url, extra_labels)
+        @url = url
+        @extra_labels = extra_labels
+        GreekFire::Measure.before_metrics { refresh }
+      end
+
+      def elasticsearch_metrics(client, index)
+        metrics = OpenStruct.new
+        metrics.index = index
+        stats = client.indices.stats["indices"][index]["total"]
+        metrics.docs_count = stats["docs"]["count"]
+        metrics.docs_deleted = stats["docs"]["deleted"]
+        metrics
+      end
+
+      def refresh
+        client = ::Elasticsearch::Client.new(hosts: @url)
+        aliases = client.indices.get_aliases
+
+        @metrics = aliases.map do |index, _aliases|
+          elasticsearch_metrics(client, index)
+        end
+      end
+
+    end
+
+    class Measure < GreekFire::Measure
+      def initialize(name, &block)
+        super(name) do |label|
+          block.call(label.delete(:metric))
+        end
+      end
+
+
+      def labels
+        ClearSkies::Elasticsearch::Report.reports.map do |report|
+          report.metrics.map do |metric|
+            report.extra_labels.merge({index: metric.index, metric: metric })
+          end
+        end.flatten
+      end
+    end
+
+    class Gauge < ClearSkies::Elasticsearch::Measure
+      def to_partial_path
+        GreekFire::Gauge._to_partial_path
+      end
+    end
+    class Counter < ClearSkies::Elasticsearch::Measure
+      def to_partial_path
+        GreekFire::Counter._to_partial_path
+      end
+    end
+  end
+end
+
+# GreekFire::Metric.register(ClearSkies::Elasticsearch::Gauge.new("elasticsearch_docs_count")                     { |metrics| metrics.docs_count })
+# GreekFire::Metric.register(ClearSkies::Elasticsearch::Gauge.new("elasticsearch_docs_deleted")                   { |metrics| metrics.docs_deleted })
+
+
+#For a <DIM>, grab N metrics.

data/lib/clear_skies/redis/report.rb

@@ -1,52 +1,117 @@
 require 'redis'
+
 module ClearSkies
   module Redis
+
     class Report
+
       def self.register(host, port, extra_labels=nil)
-        extra_labels = Hash.new unless extra_labels
-
-        r = ClearSkies::Redis::Report.new(host, port)
-        databases = Proc.new { ::Redis.new(:host => host, :port => port).info.keys.map {|k| k =~ /^db/ && k.sub("db", "")}.compact }
-
-        GreekFire::Gauge.register("redis_keys",                        labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).keys }
-        GreekFire::Gauge.register("redis_last_save",                   labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).last_save }
-        GreekFire::Gauge.register("redis_uptime",                      labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).uptime }
-        GreekFire::Gauge.register("redis_connected_clients",           labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).connected_clients }
-        GreekFire::Gauge.register("redis_blocked_clients",             labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).blocked_clients }
-        GreekFire::Gauge.register("redis_used_memory",                 labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).used_memory }
-        GreekFire::Gauge.register("redis_mem_fragmentation_ratio",     labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).mem_fragmentation_ratio }
-        GreekFire::Gauge.register("redis_rdb_changes_since_last_save", labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).rdb_changes_since_last_save }
-        GreekFire::Gauge.register("redis_rdb_last_bgsave_time_sec",    labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).rdb_last_bgsave_time_sec }
-        GreekFire::Counter.register("redis_total_commands_processed",  labels: {"database" => databases} ) { | labels| labels.merge!(extra_labels);r.redis_metrics(labels).total_commands_processed }
-      end
-
-
-      def redis_metrics(labels)
-        cache_key = "redis_stats_#{@host}_#{@port}_#{labels["database"]}"
-        Rails.cache.fetch(cache_key, expires_in: 1.second) do
-          redis = ::Redis.new(host: @host, port: @port, db: labels["database"])
-          redis_info = redis.info
-          metrics = OpenStruct.new
-
-          metrics.keys = redis.dbsize
-          metrics.last_save = Time.now.to_i - redis.lastsave
-          metrics.uptime = redis_info["uptime_in_seconds"].to_f
-          metrics.connected_clients =  redis_info["connected_clients"].to_i
-          metrics.blocked_clients =  redis_info["blocked_clients"].to_i
-          metrics.used_memory =  redis_info["used_memory"].to_f
-          metrics.mem_fragmentation_ratio =  redis_info["mem_fragmentation_ratio"].to_f
-          metrics.rdb_changes_since_last_save =  redis_info["rdb_changes_since_last_save"].to_f
-          metrics.rdb_last_bgsave_time_sec =  redis_info["rdb_last_bgsave_time_sec"].to_f
-          metrics.total_commands_processed =  redis_info["total_commands_processed"].to_f
-          metrics
-        end
+        reports << ClearSkies::Redis::Report.new(host, port, extra_labels || Hash.new)
+      end
+
+      def self.reports
+        @reports ||= []
       end
 
-      private
-      def initialize(host, port)
+      attr_reader :host, :port, :extra_labels
+      def initialize(host, port, extra_labels)
         @host = host
         @port = port
+        @extra_labels = {host: host, port: port}.merge extra_labels
+      end
+
+      def dimensions
+        ::Redis.new(:host => @host, :port => @port).info.keys.map {|k| k =~ /^db/ && k.sub("db", "")}.compact
+      end
+
+      def report_dimensions
+        dimensions.map do |dimension|
+          ReportDimension.new(self, dimension)
+        end
+      end
+
+      def items
+        report_dimensions.map
+      end
+    end
+
+    class MeasureSet < GreekFire::MeasureSet
+      def items
+        report_dimensions = ClearSkies::Redis::Report.reports.map(&:report_dimensions).flatten
+        return [] unless report_dimensions.length > 0
+
+        [
+            "keys",
+            "last_save",
+            "uptime",
+            "connected_clients",
+            "blocked_clients",
+            "used_memory",
+            "mem_fragmentation_ratio",
+            "rdb_changes_since_last_save",
+            "rdb_last_bgsave_time_sec",
+            "total_commands_processed"
+        ].map do |metric_name|
+          ClearSkies::Redis::Gauge.new(report_dimensions,  "redis", metric_name)
+        end
+      end
+    end
+
+    class ReportDimension
+      attr_reader :report, :dimension
+
+      def initialize(report, dimension)
+        @report = report
+        @dimension = dimension
+      end
+
+      def metrics
+        redis = ::Redis.new(host: report.host, port: report.port, db: @dimension)
+        redis_info = redis.info
+
+        metrics = OpenStruct.new
+        metrics.keys = redis.dbsize
+        metrics.last_save = Time.now.to_i - redis.lastsave
+        metrics.uptime = redis_info["uptime_in_seconds"].to_f
+        metrics.connected_clients =  redis_info["connected_clients"].to_i
+        metrics.blocked_clients =  redis_info["blocked_clients"].to_i
+        metrics.used_memory =  redis_info["used_memory"].to_f
+        metrics.mem_fragmentation_ratio =  redis_info["mem_fragmentation_ratio"].to_f
+        metrics.rdb_changes_since_last_save =  redis_info["rdb_changes_since_last_save"].to_f
+        metrics.rdb_last_bgsave_time_szec =  redis_info["rdb_last_bgsave_time_sec"].to_f
+        metrics.total_commands_processed =  redis_info["total_commands_processed"].to_f
+        metrics
+      end
+    end
+
+    class Measure < GreekFire::Measure
+      def initialize(report_dimensions, prefix, name)
+        @report_dimensions = report_dimensions
+        super("#{prefix}_#{name}") do |label|
+          label.delete(:metric).metrics.to_h[name.to_sym]
+        end
+      end
+
+
+      def labels
+        @report_dimensions.map do |report_dimension|
+          report_dimension.report.extra_labels.merge({metric: report_dimension })
+        end
+      end
+    end
+
+    class Gauge < ClearSkies::Redis::Measure
+      def to_partial_path
+        GreekFire::Gauge._to_partial_path
+      end
+    end
+    class Counter < ClearSkies::Redis::Measure
+      def to_partial_path
+        GreekFire::Counter._to_partial_path
       end
     end
   end
 end
+
+GreekFire::Metric.register ClearSkies::Redis::MeasureSet.new
+

data/lib/clear_skies/version.rb

@@ -1,3 +1,3 @@
 module ClearSkies
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/clear_skies.rb

@@ -7,9 +7,12 @@ require "greek_fire"
 require "aws-sdk"
 
 require "clear_skies/app"
-require "clear_skies/cloud_watch/gauge"
-require "clear_skies/cloud_watch/rds_gauge"
-require "clear_skies/cloud_watch/elb_gauge"
-require "clear_skies/cloud_watch/elastic_beanstalk_gauge"
-require "clear_skies/cloud_watch/request_counter"
-require "clear_skies/redis/report"
+require "clear_skies/aws/reservation_utilization"
+require "clear_skies/aws/cloud_watch/gauge"
+require "clear_skies/aws/cloud_watch/billing"
+require "clear_skies/aws/cloud_watch/rds_gauge"
+require "clear_skies/aws/cloud_watch/elb_gauge"
+require "clear_skies/aws/cloud_watch/elastic_beanstalk_gauge"
+require "clear_skies/aws/cloud_watch/request_counter"
+require "clear_skies/redis/report"
+require "clear_skies/elasticsearch/report"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clear_skies
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Chris Constantine
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-23 00:00:00.000000000 Z
+date: 2017-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puma
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: greek_fire
   requirement: !ruby/object:Gem::Requirement
@@ -121,6 +135,7 @@ files:
 - ".ruby-version"
 - Dockerfile
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -130,11 +145,14 @@ files:
 - exe/clear_skies
 - lib/clear_skies.rb
 - lib/clear_skies/app.rb
-- lib/clear_skies/cloud_watch/elastic_beanstalk_gauge.rb
-- lib/clear_skies/cloud_watch/elb_gauge.rb
-- lib/clear_skies/cloud_watch/gauge.rb
-- lib/clear_skies/cloud_watch/rds_gauge.rb
-- lib/clear_skies/cloud_watch/request_counter.rb
+- lib/clear_skies/aws/cloud_watch/billing.rb
+- lib/clear_skies/aws/cloud_watch/elastic_beanstalk_gauge.rb
+- lib/clear_skies/aws/cloud_watch/elb_gauge.rb
+- lib/clear_skies/aws/cloud_watch/gauge.rb
+- lib/clear_skies/aws/cloud_watch/rds_gauge.rb
+- lib/clear_skies/aws/cloud_watch/request_counter.rb
+- lib/clear_skies/aws/reservation_utilization.rb
+- lib/clear_skies/elasticsearch/report.rb
 - lib/clear_skies/redis/report.rb
 - lib/clear_skies/version.rb
 homepage: https://github.com/omadahealth/clear_skies

data/lib/clear_skies/cloud_watch/elastic_beanstalk_gauge.rb

@@ -1,45 +0,0 @@
-module ClearSkies
-  module CloudWatch
-    class ElasticBeanstalkGauge < ClearSkies::CloudWatch::Gauge
-      def self.client
-        @client ||= Aws::ElasticBeanstalk::Client.new
-      end
-      def initialize(metric_name, dimension, statistics, description: nil, &block)
-        super("AWS/ElasticBeanstalk", metric_name, dimension, statistics, description: description, &block)
-      end
-
-      def application_name(environment_name)
-        ElasticBeanstalkGauge.client.describe_environments({environment_names: [environment_name] }).environments.first&.application_name
-      end
-
-      def vpc_id(application_name, environment_name)
-        config = ElasticBeanstalkGauge.client.describe_configuration_settings({ application_name: application_name, environment_name: environment_name }).
-            configuration_settings.find { |config| config.application_name == application_name && config.environment_name == environment_name}
-        option =  config.option_settings.find { |option| option.namespace == "aws:ec2:vpc" && option.option_name == "VPCId"}
-        option.value if option
-      end
-
-
-      def labels_from_metric(metric)
-        labels = super(metric)
-
-        if labels.has_key?( "environment_name") && !(Rails.cache.fetch("#{labels["environment_name"]}_skip"))
-          application_name = Rails.cache.fetch("#{labels["environment_name"]}_application_name", expires_in: 1.hour) do
-            application_name(labels["environment_name"])
-          end
-          if application_name
-            labels["application_name"] =  application_name
-
-            labels["vpc_id"] = Rails.cache.fetch("#{labels["environment_name"]}_vpc_id_") do
-              vpc_id(labels["application_name"], labels["environment_name"])
-            end
-          end
-        else
-          Rails.cache.write("#{labels["environment_name"]}_skip", true, expires_in: 1.hour, race_condition_ttl: 60.seconds)
-        end
-
-        return labels
-      end
-    end
-  end
-end

data/lib/clear_skies/cloud_watch/elb_gauge.rb

@@ -1,44 +0,0 @@
-module ClearSkies
-  module CloudWatch
-    class ELBGauge < ClearSkies::CloudWatch::Gauge
-      def self.client
-        @client ||= Aws::ElasticLoadBalancing::Client.new
-      end
-      def initialize(metric_name, dimension, statistics, description: nil, &block)
-        super("AWS/ELB", metric_name, dimension, statistics, description: description, &block)
-      end
-
-      def tags(load_balancer_name)
-        labels = {}
-        ELBGauge.client.
-            describe_tags({load_balancer_names: [load_balancer_name]}).
-            tag_descriptions.
-            select {|doc| doc.load_balancer_name == load_balancer_name}.each do |tag_description|
-          tag_description.tags.each do |tag|
-            labels[tag.key.downcase] = tag.value
-          end
-        end
-        labels
-      end
-
-      def labels_from_metric(metric)
-        labels = super(metric)
-
-        if labels.has_key?( "load_balancer_name") && !(Rails.cache.fetch("#{labels["load_balancer_name"]}_skip"))
-
-          labels["vpc_id"] = Rails.cache.fetch("#{labels["load_balancer_name"]}_vpc_id_") do
-            ELBGauge.client.describe_load_balancers(load_balancer_names: [labels["load_balancer_name"]]).load_balancer_descriptions.first.vpc_id
-          end
-
-          labels.merge!(Rails.cache.fetch("#{labels["load_balancer_name"]}_tags_", expires_in: 1.hour, race_condition_ttl: 60.seconds) do
-            tags(labels["load_balancer_name"])
-          end)
-        end
-        labels
-      rescue Aws::ElasticLoadBalancing::Errors::LoadBalancerNotFound
-        Rails.cache.write("#{labels["load_balancer_name"]}_skip", true, expires_in: 1.hour, race_condition_ttl: 60.seconds)
-        return labels
-      end
-    end
-  end
-end

data/lib/clear_skies/cloud_watch/gauge.rb

@@ -1,70 +0,0 @@
-
-module ClearSkies
-  module CloudWatch
-    class Gauge < GreekFire::Gauge
-      include ActiveModel::Conversion
-
-      def self.register(*args, &block)
-        GreekFire::Metric.register(self.new(*args, &block))
-      end
-
-      def initialize(namespace, metric_name, dimensions, statistics, description:nil, &block)
-        super("#{namespace.underscore.gsub("/", "_")}_#{metric_name.underscore}", description: description)
-        @namespace = namespace
-        @metric_name = metric_name
-        @dimensions = dimensions
-        @statistics = statistics.select { |stat| ["SampleCount", "Average", "Sum", "Minimum", "Maximum"].include?(stat.to_s) }
-        @extended_statistics = statistics - @statistics
-
-        @block = block
-      end
-
-      def self.cloudwatch_client
-        @client ||= Aws::CloudWatch::Client.new
-      end
-
-      def aws_metrics
-         Aws::CloudWatch::Resource.new(client: Gauge.cloudwatch_client).metrics(
-            namespace: @namespace,
-            metric_name: @metric_name,
-            dimensions: @dimensions.map {|dimension| {name: dimension} }
-        ).select { |metrics| metrics.dimensions.count == @dimensions.count }
-      end
-
-      def labels_from_metric(metric)
-        metric.dimensions.inject(ActiveSupport::HashWithIndifferentAccess.new) do |labels, dimension|
-          labels[dimension.name.underscore] = dimension.value
-          labels
-        end
-      end
-
-      def metrics
-        aws_metrics.map do |metric|
-          labels = labels_from_metric(metric)
-
-          next unless @block.call(labels) if @block
-
-          stats = metric.get_statistics(
-              start_time: Time.now.advance(minutes: -6),
-              end_time: Time.now.advance(minutes: -5),
-              period: 1,
-              statistics: @statistics,
-              extended_statistics: @extended_statistics,
-              dimensions: metric.dimensions
-          )
-
-          stats.datapoints.map do |datapoint|
-            datapoint.to_h.select {|k, v| ![:unit, :timestamp].include?(k) }.map do |key, value|
-              if (key == :extended_statistics)
-                value.map {|e_key, e_value| GreekFire::Metric.new(name, labels.merge({statistic: e_key}), e_value)}
-              else
-                GreekFire::Metric.new(name, labels.merge({statistic: key}), value)
-              end
-            end
-          end
-
-        end.flatten.compact
-      end
-    end
-  end
-end

data/lib/clear_skies/cloud_watch/rds_gauge.rb

@@ -1,37 +0,0 @@
-module ClearSkies
-  module CloudWatch
-    class RDSGauge < ClearSkies::CloudWatch::Gauge
-      def initialize(metric_name, dimension, statistics, description: nil, &block)
-        super("AWS/RDS", metric_name, dimension, statistics, description: description, &block)
-      end
-
-      def tags(db)
-        labels = {}
-        db.client.list_tags_for_resource(resource_name: db.db_instance_arn).tag_list.each do |tag|
-          labels[tag.key.downcase] = tag.value
-        end
-        labels
-      end
-
-      def labels_from_metric(metric)
-        labels = super(metric)
-
-        if labels.has_key?( "db_instance_identifier") && !(Rails.cache.fetch("#{labels["db_instance_identifier"]}_skip"))
-          db = Aws::RDS::DBInstance.new(labels["db_instance_identifier"])
-
-          labels["vpc_id"] = Rails.cache.fetch("#{labels["db_instance_identifier"]}_vpc_id_") do
-            db.db_subnet_group.vpc_id
-          end
-
-          labels.merge!(Rails.cache.fetch("#{labels["db_instance_identifier"]}_tags_", expires_in: 1.hour, race_condition_ttl: 60.seconds) do
-            tags(db)
-          end)
-        end
-        labels
-      rescue Aws::RDS::Errors::DBInstanceNotFound
-        Rails.cache.write("#{labels["db_instance_identifier"]}_skip", true, expires_in: 1.hour, race_condition_ttl: 60.seconds)
-        return labels
-      end
-    end
-  end
-end

data/lib/clear_skies/cloud_watch/request_counter.rb

@@ -1,37 +0,0 @@
-module ClearSkies
-  module CloudWatch
-    class RequestCounter < Seahorse::Client::Handler
-      @mutex = Mutex.new
-      def self.increment
-        @mutex.synchronize do
-          @count ||= 0
-          @count += 1
-        end
-      end
-
-      def self.count
-        @count ||= 0
-        @count
-      end
-
-      def initialize(handler)
-        @handler = handler
-      end
-
-      def call(context)
-        RequestCounter.increment
-        @handler.call(context)
-      end
-    end
-
-    class RequestCounterPlugin < Seahorse::Client::Plugin
-
-      def add_handlers(handlers, config)
-        handlers.add(RequestCounter, step: :validate)
-      end
-
-    end
-  end
-end
-
-Aws::CloudWatch::Client.add_plugin(ClearSkies::CloudWatch::RequestCounterPlugin)