rearview 1.1.2-jruby → 1.2.0-jruby

data/lib/graphite/client.rb

@@ -0,0 +1,44 @@
+module Graphite
+  class Client
+    extend Forwardable
+    attr_reader :connection
+    def_delegators :@connection, :get, :post
+    def initialize(options={})
+      default_options = {}
+      default_options[:ssl] = {
+        verify: true,
+        ca_file: File.expand_path('../cacerts.pem', __FILE__)
+      }
+      basic_auth = options.delete(:basic_auth)
+      @connection = Faraday.new(default_options.merge(options))
+      if basic_auth.present?
+        @connection.basic_auth(basic_auth[:user],basic_auth[:password])
+      end
+    end
+    def render(params={})
+      connection.get('/render',params)
+    end
+    def find_metric(metric)
+      connection.get('/metrics/find',{ query: metric })
+    end
+    def metric_exists?(metric)
+      exists = false
+      response = find_metric(metric)
+      # Does this need to handle redirects?
+      if response.status == 200 && response.headers['content-type']=='application/json'
+        json = JSON.parse(response.body)
+        exists = true unless json.empty?
+      end
+      exists
+    end
+    def reachable?
+      reachable = false
+      begin
+        response = connection.get('/render')
+        reachable = response.status==200 && response.headers['content-type']=='image/png' && response.headers['content-length'].to_i > 0
+      rescue Exception => e
+      end
+      reachable
+    end
+  end
+end

data/lib/graphite/graph.rb

@@ -0,0 +1,35 @@
+module Graphite
+  class Graph
+    attr_accessor :targets,:params
+    def initialize
+      @targets = []
+      @params = {}
+    end
+    def method_missing(sym)
+      if @params.has_key?(sym.to_s)
+        @params[sym.to_s]
+      else
+        super
+      end
+    end
+    def self.from_url(url)
+      graph = self.new
+      uri = URI(url)
+      url_params = CGI.parse(uri.query)
+      target_parser = Graphite::TargetParser.new
+      url_params.delete("target").each do |t|
+        target_parser.parse!(t)
+        graph.targets << target_parser.tree.to_model
+      end
+      url_params.keys.inject(graph.params) { |acc,key|
+        acc[key] = if url_params[key].present? && url_params[key].size==1
+                     url_params[key].first
+                   else
+                     url_params[key]
+                   end
+        acc
+      }
+      graph
+    end
+  end
+end

data/lib/{rearview/graphite_parser.rb → graphite/raw_parser.rb}

@@ -1,11 +1,10 @@
-module Rearview
-  class GraphiteParser
+module Graphite
+  class RawParser
     ## returns an array of arrays of pairs with 2 elements timestamp and value (may be nil)
     def self.parse(lines)
       lines.strip.split("\n").map do |line|
         metric, startTime, endTime, interval, dataStr = /(.*),(\d+),(\d+),(\d+)\|(.*)/.match(line)[1..-1]
         data = dataStr.split(",")
-
         0.upto(data.length - 1).map do |i|
           value = data[i]
           {

data/lib/graphite/target.rb

@@ -0,0 +1,9 @@
+module Graphite
+  class Target
+    attr_accessor :color,:alias,:metric,:functions
+    def initialize
+      yield self if block_given?
+    end
+  end
+end
+

data/lib/graphite/target_grammer.rb

@@ -0,0 +1,114 @@
+module Graphite
+  module TargetGrammer
+
+    class SyntaxNode < Treetop::Runtime::SyntaxNode
+      def text
+        self.text_value
+      end
+    end
+
+    class Target < SyntaxNode
+      def path?
+        elements[0].kind_of?(Graphite::TargetGrammer::Path)
+      end
+      def expression?
+        elements[0].kind_of?(Graphite::TargetGrammer::Expression)
+      end
+      def comment?
+        elements[0].kind_of?(Graphite::TargetGrammer::Comment)
+      end
+      def unknown?
+        !path? && !expression?
+      end
+      def color
+        unless @color
+          e = self.expressions.detect {|e| e.identifier.try(:text) == "color" }
+          if e && e.args && e.args.elements
+            color_val = e.args.elements.last.detect { |e| e.kind_of?(Graphite::TargetGrammer::StringLiteral)  }
+            unless color_val.nil?
+              @color = color_val.text
+            end
+          end
+        end
+        @color
+      end
+      def alias
+        unless @alias
+          e = self.expressions.detect {|e| e.identifier.try(:text) == "alias" }
+          if e && e.args && e.args.elements
+            alias_val = e.args.elements.last.detect { |e| e.kind_of?(Graphite::TargetGrammer::StringLiteral) }
+            unless alias_val.nil?
+              @alias = alias_val.text
+            end
+          end
+        end
+        @alias
+      end
+      def metric
+        unless @metric
+          @metric = if path?
+            self.elements[0].text
+          else
+            path = self.detect { |e| e.kind_of?(Graphite::TargetGrammer::Path) }
+            unless path.nil?
+              path.text
+            end
+          end
+        end
+        @metric
+      end
+      def functions
+        unless @functions
+          @functions = self.expressions.map { |e| e.identifier.text }
+        end
+        @functions
+      end
+      def expressions
+        unless @expressions
+          @expressions = self.find_all { |e| e.kind_of?(Graphite::TargetGrammer::Expression) }
+        end
+        @expressions
+      end
+      def to_model
+        Graphite::Target.new do |model|
+          model.alias = self.alias
+          model.color = self.color
+          model.metric = self.metric
+          model.functions = self.functions
+        end
+      end
+    end
+
+    class IntegerLiteral < SyntaxNode
+    end
+
+    class StringLiteral < SyntaxNode
+      def text
+        eval self.text_value
+      end
+    end
+
+    class FloatLiteral < SyntaxNode
+    end
+
+    class Identifier < SyntaxNode
+    end
+
+    class Expression < SyntaxNode
+    end
+
+    class PathSegment < SyntaxNode
+    end
+
+    class Path < SyntaxNode
+    end
+
+    class Arg < SyntaxNode
+    end
+
+    class Comment < SyntaxNode
+    end
+
+  end
+end
+

data/lib/graphite/target_grammer.treetop

@@ -0,0 +1,53 @@
+grammar TargetGrammer
+
+  rule target
+    ( comment / expression / path )* <Graphite::TargetGrammer::Target>
+  end
+
+  rule expression
+    identifier '(' args ')' <Graphite::TargetGrammer::Expression>
+  end
+
+  rule args
+    arg ( space* ',' space* arg )*
+  end
+
+  rule arg
+    ( expression / path / string / integer / float / space )?
+  end
+
+  rule path
+    path_segment ( '.' path_segment )* <Graphite::TargetGrammer::Path>
+  end
+
+  rule path_segment
+    [a-zA-Z0-9_\*\-]+ <Graphite::TargetGrammer::PathSegment>
+  end
+
+  rule comment
+    '#' ( . )* <Graphite::TargetGrammer::Comment>
+  end
+
+  rule integer
+    ('+' / '-')? [0-9]+ <Graphite::TargetGrammer::IntegerLiteral>
+  end
+
+  rule float
+    ('+' / '-')? [0-9]+ (('.' [0-9]+) / ('e' [0-9]+)) <Graphite::TargetGrammer::FloatLiteral>
+  end
+
+  rule string
+    '"' ('\"' / !'"' .)* '"' <Graphite::TargetGrammer::StringLiteral> 
+    / 
+    "'" ('\"' / !"'" .)* "'" <Graphite::TargetGrammer::StringLiteral>
+  end
+
+  rule identifier
+    [a-zA-Z0-9_]* <Graphite::TargetGrammer::Identifier>
+  end
+
+  rule space
+    [\s]+
+  end
+
+end

data/lib/graphite/target_parser.rb

@@ -0,0 +1,50 @@
+
+module Graphite
+  class TargetParser
+    class TargetParserError < StandardError
+      attr_accessor :parser_details
+      def initialize(parser_details)
+        @parser_details = parser_details
+      end
+      def to_s
+        "parse failure %{message}" % parser_details
+      end
+    end
+    attr_reader :parser,:tree
+    attr_accessor :data
+    def initialize(data=nil)
+      @data = data
+      @parser = TargetGrammerParser.new
+    end
+    def parse(data=nil)
+      if data.present?
+        @data = data
+      end
+      @tree = @parser.parse(@data)
+    end
+    def parse!(data=nil)
+      self.parse(data)
+      if error?
+        raise TargetParserError.new(error)
+      end
+      @tree
+    end
+    def self.parse(data)
+      inst = self.new(data)
+      inst.parse
+      inst
+    end
+    def error?
+      @tree.nil?
+    end
+    def error
+      { message: @parser.failure_reason, line: @parser.failure_line, column: @parser.failure_column }
+    end
+    def self.grammer
+      File.expand_path("../target_grammer.treetop",__FILE__)
+    end
+  end
+end
+
+Treetop.load(Graphite::TargetParser.grammer)
+

data/lib/graphite/treetop_ext.rb

@@ -0,0 +1,14 @@
+
+class Treetop::Runtime::SyntaxNode
+  include Enumerable
+  def each(&block)
+    self.elements.each { |e| each_recursive(e,&block) }
+  end
+  def each_recursive(node,&block)
+    yield node
+    unless node.terminal?
+      node.elements.each { |e| each_recursive(e,&block) }
+    end
+  end
+end
+

data/lib/rearview.rb

@@ -1,3 +1,5 @@
+require 'graphite'
+
 require 'rearview/engine'
 require 'rearview/concerns'
 require 'rearview/ext/state_machine'
@@ -6,7 +8,6 @@ require 'rearview/constants_module_maker'
 require 'rearview/logger'
 require 'rearview/log_formatter'
 require 'rearview/cron_helper'
-require 'rearview/graphite_parser'
 require 'rearview/results_handler'
 require 'rearview/alerts_handler'
 require 'rearview/url_helper'
@@ -16,6 +17,10 @@ require 'rearview/monitor_task'
 require 'rearview/distribute'
 require 'rearview/monitor_supervisor'
 require 'rearview/monitor_service'
+require 'rearview/metrics_validator'
+require 'rearview/metrics_validator_task'
+require 'rearview/metrics_validator_service'
+require 'rearview/cron_expression_validator'
 require 'rearview/configuration'
 require 'rearview/vm'
 require 'rearview/statsd'
@@ -27,7 +32,7 @@ module Rearview
   include Rearview::Logger
 
   class << self
-    attr_accessor :monitor_service,:stats_service,:alert_clients
+    attr_accessor :monitor_service,:stats_service,:validator_service,:alert_clients
   end
 
   module_function
@@ -70,10 +75,13 @@ module Rearview
       logger.warn "monitor disabled!"
     end
     if config.stats_enabled?
-      logger.info "starting up stats service"
       @stats_service = Rearview::StatsService.supervise
       @stats_service.actors.first.startup
     end
+    if config.metrics_validator_enabled?
+      @validator_service = Rearview::MetricsValidatorService.supervise
+      @validator_service.actors.first.startup
+    end
     @alert_clients = Rearview::Alerts.registry.values
     @booted = true
   end

data/lib/rearview/alerts_handler.rb

@@ -8,21 +8,17 @@ module Rearview
       @monitor_results = monitor_results
     end
     def run
-      logger.info "#{self} run"
       if Rearview.config.alerts_enabled?
         Rearview.alert_clients.each do |client|
-          alert_agent = client.new
           begin
+            alert_agent = client.new
             alert_agent.alert(@job,@monitor_results)
           rescue
-            logger.error "#{self} #{alert_agent} failed: #{$!}\n#{$@.join("\n")}"
+            logger.error "#{self} #{client} failed: #{$!}\n#{$@.join("\n")}"
           end
         end
       end
       self
-    rescue
-      logger.error "#{self} failed: #{$!}\n#{$@.join("\n")}"
-      self
     end
   end
 end

data/lib/rearview/configuration.rb

@@ -8,49 +8,39 @@ module Rearview
 
     class UrlValidator < ActiveModel::EachValidator
       def validate_each(record, attribute, value)
+        passed = true
         if value.present?
           uri = URI.parse(value) rescue nil
           unless uri.present? && uri.scheme.present? && (uri.scheme.downcase=="http" || uri.scheme.downcase=="https")
             record.errors.add attribute, (options[:message] || "is not a valid URL")
-          else
-            if options[:reachable]
-              reachable = ReachableValidator.new(options.dup.merge(attributes: @attributes))
-              reachable.validate_each(record,attribute,value)
-            end
-          end
-        end
-      end
-    end
-
-    class ReachableValidator < ActiveModel::EachValidator
-      def validate_each(record, attribute, value)
-        if value.present?
-          response = HTTParty.get(value) rescue nil
-          unless response.present? && response.code == 200
-            record.errors.add attribute, (options[:message] || "is not a reachable URL")
+            passed = false
           end
         end
+        passed
       end
     end
 
     class DirectoryValidator < ActiveModel::EachValidator
       def validate_each(record, attribute, value)
+        passed = true
         if value.present?
           unless File.directory?(value)
             record.errors.add attribute, (options[:message] || "is not a directory")
+            passed = false
           end
         end
       end
     end
 
-    ATTRIBUTES = [:default_from, :graphite_url, :pagerduty_url, :sandbox_exec,
+    ATTRIBUTES = [:default_from, :graphite_connection, :pagerduty_url, :sandbox_exec,
                   :sandbox_timeout, :sandbox_dir, :enable_alerts, :preload_jobs,
                   :logger, :enable_monitor, :verify, :default_url_options,
-                  :authentication, :enable_stats, :statsd_connection]
+                  :authentication, :enable_stats, :statsd_connection,
+                  :enable_metrics_validator, :metrics_validator_schedule]
 
     attr_accessor *ATTRIBUTES
 
-    validates :graphite_url, presence: true, url: { reachable: true }
+    validates :graphite_connection, presence: true
     validates :pagerduty_url, presence: true, url: true
     validates :default_from, presence: true
     validates :default_url_options, presence: true
@@ -59,8 +49,10 @@ module Rearview
     validates :sandbox_timeout, presence: true, numericality: { greater_than: 4 }
     validates :authentication, presence: true
     validates :statsd_connection, presence: true, if: -> { self.stats_enabled? }
+    validates :metrics_validator_schedule, presence: true, :'rearview/cron_expression' => true, if: -> { self.metrics_validator_enabled? }
 
     validate :validate_sandbox_execution
+    validate :validate_graphite_connection
 
     def initialize(attributes={})
       @default_from = "rearview@localhost"
@@ -73,6 +65,9 @@ module Rearview
       @enable_stats = false
       @default_url_options = {:host=>"localhost",:port=>"3000"}
       @pagerduty_url = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+      @graphite_connection = {}
+      @enable_metrics_validator = false
+      @metrics_validator_schedule = nil
       super
     end
 
@@ -84,6 +79,10 @@ module Rearview
       enable_monitor
     end
 
+    def metrics_validator_enabled?
+      enable_metrics_validator
+    end
+
     def preload_jobs?
       preload_jobs
     end
@@ -102,6 +101,16 @@ module Rearview
         opts.on("--[no-]alerts", "Enable/disable alerts")  { |v| self.enable_alerts = v }
         opts.on("--[no-]monitor", "Enable/disable monitor")  { |v| self.enable_monitor = v }
         opts.on("--[no-]verify", "Enable/disable verification")  { |v| self.verify = v }
+        opts.on("--[no-]stats-service", "Enable/disable stats service")  { |v| self.enable_stats = v }
+        opts.on("--[no-]validation-service", "Enable/disable validation service")  { |v| self.enable_metrics_validator = v }
+        opts.on("--[no-]soft-boot", "Enable/disable soft boot") { |v|
+          if v 
+            self.preload_jobs = false
+            self.enable_stats = false
+            self.enable_metrics_validator = false
+            self.enable_alerts = false
+          end
+        }
       end.parse!(argv)
     end
 
@@ -118,6 +127,22 @@ module Rearview
        exit_code == 0
     end
 
+    def validate_graphite_connection
+      if graphite_connection.present?
+        if !graphite_connection[:url].present?
+          self.errors.add(:graphite_connection, "graphite URL can't be blank")
+        else
+          url_validator = UrlValidator.new({ attributes: [:graphite_connection], message: "does not contain a valid URL" })
+          if url_validator.validate_each(self,:graphite_connection,graphite_connection[:url])
+            client = Graphite::Client.new(graphite_connection)
+            unless(client.reachable?)
+              self.errors.add(:graphite_connection, "graphite cannot be reached")
+            end
+          end
+        end
+      end
+    end
+
     def dump
       ATTRIBUTES.sort.map { |attrib| "#{attrib.to_s}=#{(self.send(attrib).nil? ? "nil" : self.send(attrib))}" }.join("\n")
     end