rearview 1.0.0-jruby

data/lib/rearview/monitor_service.rb

@@ -0,0 +1,70 @@
+
+module Rearview
+  class MonitorService
+    class MonitorServiceError < StandardError; end;
+    include Celluloid
+    include Rearview::Logger
+    attr_accessor :jobs
+    attr_reader :supervisor
+    def initialize(jobs=[])
+      self.jobs = jobs
+      @supervisor = nil
+      @started = false
+    end
+    def jobs=(jobs)
+      @jobs = jobs.inject({}) { |c,v| c[v.id] = v ; c }
+    end
+    def started?
+      @started
+    end
+    def startup
+      raise MonitorServiceError.new("service already started") if started?
+      # TODO actor could die, need to reference by name in registry and/or create link
+      @supervisor = Rearview::MonitorSupervisor.run!
+      @supervisor.add_tasks(@jobs.values)
+      @started = true
+    end
+    def shutdown
+      raise MonitorServiceError.new("service not started") unless started?
+      @supervisor.remove_all_tasks
+      @supervisor.terminate
+      @started = false
+    end
+    def schedule(job)
+      logger.debug "#{self} schedule job: #{job.id}"
+      raise MonitorServiceError.new("service not started") unless started?
+      if @jobs[job.id]
+        remove(job)
+      end
+      add(job)
+    end
+    def unschedule(job)
+      logger.debug "#{self} unschedule job: #{job.id}"
+      raise MonitorServiceError.new("service not started") unless started?
+      if @jobs[job.id]
+        remove(job)
+      end
+    end
+    def add(job)
+      raise MonitorServiceError.new("service not started") unless started?
+      if @jobs[job.id]
+        logger.warn "#{self}#add job:#{job.id} already added"
+      else
+        @supervisor.add_tasks([job])
+        @jobs[job.id] = job
+      end
+    end
+    def remove(job)
+      raise MonitorServiceError.new("service not started") unless started?
+      if !@jobs[job.id]
+        logger.warn "#{self}#remove job:#{job.id} has already been removed"
+      else
+        @supervisor.remove_tasks([job])
+        @jobs.delete(job.id)
+      end
+    end
+
+  end
+
+end
+

data/lib/rearview/monitor_supervisor.rb

@@ -0,0 +1,54 @@
+
+module Rearview
+  class MonitorSupervisor < Celluloid::SupervisionGroup
+    include Rearview::Logger
+    class MonitorSupervisorError < StandardError; end;
+    def add_tasks(jobs)
+      if jobs.present?
+        dist_jobs = Rearview::Distribute.by_delay(jobs)
+        logger.debug Rearview::Distribute.inspect(dist_jobs)
+        dist_jobs.each do |dj|
+          job = dj.first
+          meta = dj.last
+          add(Rearview::MonitorTask,as: MonitorSupervisor.task_sym(job), args: [job,meta[:initial_delay]])
+        end
+      end
+    end
+    def remove_tasks(jobs)
+      if jobs.present?
+        jobs.each do |j|
+          logger.debug "#{self} removing job:#{j.id}"
+          member = @members.find { |m| m.actor.job == j }
+          if !member.present?
+            warn "#{self} remove job:#{j.id} failed because it is not present"
+          else
+            @members.reject! { |m| m == member }
+            unlink(member.actor)
+            member.actor.terminate
+          end
+        end
+      end
+    end
+    def all_tasks
+      actors
+    end
+    def remove_all_tasks
+      actors.each do |a|
+        begin
+          unlink(a)
+          a.terminate
+        rescue DeadActorError, MailboxError
+          warn "#{self} error while terminating #{a}: #{$!}\n#{$@}"
+        end
+      end
+      @members = []
+    end
+    def self.task_sym(job)
+      "job_#{job.id}".to_sym
+    end
+    def to_s
+      "#{super.to_s} [threadId:#{java.lang.Thread.currentThread.getId} threadName:#{java.lang.Thread.currentThread.getName}]"
+    end
+  end
+end
+

data/lib/rearview/monitor_task.rb

@@ -0,0 +1,50 @@
+require 'json'
+
+module Rearview
+  class MonitorTask
+    class MonitorTaskError < StandardError; end;
+    include Celluloid
+    include Celluloid::Logger
+    attr_reader :job,:timer,:initial_delay
+    def initialize(job,initial_delay=0,start=true)
+      debug "#{self} intialize initial_delay:#{initial_delay} start:#{start}"
+      @job = job
+      @initial_delay = initial_delay
+      schedule if start
+    end
+    def schedule
+      debug "#{self} schedule"
+      if ActiveRecord::Base.connection_pool.active_connection?
+        ActiveRecord::Base.connection_pool.release_connection
+      end
+      # TODO is this really necessary?
+      if(@timer)
+        @timer.cancel
+      end
+      delay = @job.delay + @initial_delay
+      debug "#{self} next run in #{delay}s"
+      @timer = after(delay) { self.run }
+    end
+    def run
+      debug "#{self} run"
+      ActiveRecord::Base.connection_pool.with_connection do
+        @initial_delay = 0
+        result = Rearview::MonitorRunner.run(@job.metrics, @job.monitor_expr, @job.minutes)
+        @job.last_run = Time.now.utc
+        Rearview::ResultsHandler.new(@job,result).run
+      end
+    rescue
+      error "#{self} run failed: #{$!}\n#{$@.join("\n")}"
+      ActiveRecord::Base.connection_pool.with_connection do
+        @job.last_run = Time.now.utc
+        @job.error!
+      end
+    ensure
+      schedule
+    end
+    def to_s
+      "#{super.to_s} [jobId:#{self.job.try(:id)||'nil'} threadId:#{java.lang.Thread.currentThread.getId} threadName:#{java.lang.Thread.currentThread.getName}]"
+    end
+  end
+end
+

data/lib/rearview/results_handler.rb

@@ -0,0 +1,38 @@
+
+module Rearview
+  class ResultsHandler
+    attr_reader :status,:job,:monitor_results
+    include Rearview::Logger
+    def initialize(job,monitor_results)
+      @job = job
+      @status = Job::Status::ERROR
+      @monitor_results = monitor_results || {}
+    end
+    def run
+      logger.info "#{self} run"
+      normalized_results = Rearview::MonitorRunner.normalize_results(@monitor_results)
+      @status = normalized_results[:status] || @status
+
+      job_data = JobData.find_or_create_by(job_id: @job.id)
+      job_data.data = normalized_results
+      job_data.save!
+
+      @job.last_run = Time.now.utc
+      event = if Job::Status.values.include?(@status.to_s)
+                @status.to_sym
+              else
+                :error
+              end
+      logger.info "#{self} firing event :#{event} for #{@job.inspect}"
+      @job.fire_event(event,{:monitor_results=>@monitor_results,:job_error=>{:message=>normalized_results[:output]}})
+
+      self
+    rescue
+      logger.error "#{self} process results failed: #{$!}\n#{$@.join("\n")}"
+      self
+    end
+  end
+  def to_s
+    "#{super.to_s} [jobId:#{@job.id} threadId:#{java.lang.Thread.currentThread.getId} threadName:#{java.lang.Thread.currentThread.getName}]"
+  end
+end

data/lib/rearview/sandbox.rb

@@ -0,0 +1,47 @@
+
+module Rearview
+  module Sandbox
+    class << self
+
+      include Rearview::Logger
+
+      def valid?
+        valid_exec = self.valid_exec?
+        valid_graphite_connection = self.valid_graphite_connection?
+        valid_exec && valid_graphite_connection
+      end
+
+      def valid_exec?
+        script_file = File.join(Rearview.config.sandbox_dir,"verify_sandbox.rb")
+        cmd = Rearview.config.sandbox_exec.clone << script_file
+        logger.info "#{self} checking sandbox with #{cmd}"
+        process_builder = ProcessBuilder.new(cmd).redirectErrorStream(true)
+        process_builder.directory(java.io.File.new(Rearview.config.sandbox_dir.to_s))
+        process_builder.environment.delete("GEM_HOME")
+        process_builder.environment.delete("GEM_PATH")
+        process = process_builder.start
+        exit_code = process.waitFor
+        output = process.get_input_stream.to_io.read
+        logger.info "#{self} sandbox execution: \n#{output}"
+        unless exit_code == 0
+          logger.error "#{self} unable to execute in sandbox"
+        end
+        exit_code == 0
+      end
+
+      def valid_graphite_connection?
+        url = "#{Rearview.config.graphite_url}/dashboard"
+        logger.info "#{self} checking graphite connection to #{url}"
+        response = HTTParty.get(url)
+        unless response.code == 200
+          logger.error "#{self} unable to communicate with graphite"
+        end
+        response.code == 200
+      rescue
+        logger.error "#{self} unable to communicate with graphite #{$!}"
+        false
+      end
+
+    end
+  end
+end

data/lib/rearview/templates/monitor.rb

@@ -0,0 +1,164 @@
+#
+# Do not edit unless you know what you're doing!!! This code affects the sandbox behavior
+# and monitor functionality.
+#
+
+require 'rubygems'
+require 'bundler'
+require 'stringio'
+
+Bundler.setup
+
+require 'json'
+%{utilities}
+
+class Metric
+  attr_reader :timestamp, :value
+
+  def initialize(label_fn, timestamp, value)
+    @label_fn  = label_fn
+    @timestamp = timestamp
+    @value     = value
+  end
+
+  def label
+     @label_fn.call
+   end
+
+  def to_s
+    "{ label: #{label}, timestamp: #{timestamp}, value: #{value.nil? ? "nil" : value.to_f} }"
+  end
+end
+
+
+class TimeSeries
+  attr_accessor :label
+  attr_reader :entries
+
+  def initialize(ts)
+    @label   = ts.first["metric"]
+    @entries = ts.map { |t| Metric.new(lambda { self.label }, t["timestamp"], t["value"]) }
+    def @entries.to_s
+      "[ #{self.join ", "} ]"
+    end
+  end
+
+  def values
+    @entries.map { |e| e.value }
+  end
+
+  def to_s
+    "{ label: #{@label}, entries: [ #{@entries.join ", "} ] }"
+  end
+end
+
+
+class Scoped
+  include MonitorUtilities
+
+  def initialize(timeout)
+    @graph_data         = {}
+    @graph_data.default = [] # make the default value an empty array
+    @timeout            = timeout
+  end
+
+  def graph_value(name, timestamp, value)
+    @graph_data[name] += [[timestamp, value]]
+  end
+
+  def fold_metrics(initial, &block)
+    iterations = @timeseries[0].values.length
+    iterations.times.inject(initial) do |accum, i|
+      entries = @timeseries.map { |series| series.entries[i] }
+      block.call accum, *entries
+    end
+  end
+
+  def with_metrics(&block)
+    iterations = @timeseries[0].values.length
+    iterations.times.each do |i|
+      entries = @timeseries.map { |series| series.entries[i] }
+      block.call *entries
+    end
+  end
+
+  def scoped_eval(namespace)
+    # Copy instance variables from top-level class into current scope
+    namespace.keys.each do |v|
+      instance_variable_set "#{v}".to_sym, namespace[v]
+    end
+
+    # Bolt on a to_s to @timeseries to pretty print the instances
+    def @timeseries.to_s
+      "[ #{self.join ", "} ]"
+    end
+
+    out = StringIO.new
+
+    t = Thread.start do
+      $stdout = out
+      $SAFE=3
+      begin
+        graph_value = lambda { |name, timestamp, value| graph_value(name, timestamp, value) }
+        %{expression}
+      rescue RuntimeError => e
+        out.write e
+        @error = e.to_s
+      end
+    end
+
+    t.abort_on_exception = false
+
+    begin
+      t.join
+    rescue => e
+      out.write e
+      @error = e.to_s
+    ensure
+      $SAFE = 0
+      $stdout = STDOUT
+      $stderr = STDERR
+    end
+
+    { :graph_data => @graph_data, :output => out.string, :error => @error}.to_json
+  end
+end
+
+
+class Wrapper
+  def create_timeseries(ts)
+    ns   = {}
+    ts   = ts.to_a.map { |t| TimeSeries.new(t) }
+    vars = (0...ts.length).map do |i|
+      varname = variable_by_offset(i)
+      ns["@#{varname}"] = ts[i]
+    end
+    ns["@timeseries"] = ts
+    ns
+  end
+
+  #
+  # Return a variable name (e.g. a,b,c) from an ordinal
+  #
+  def variable_by_offset(index)
+    offset = 'a'.ord
+    dv  = index.divmod(26)
+    ext = dv[0] > 0 ? dv[1].to_s : ""
+    (offset + dv[1]).chr + ext
+  end
+
+
+  # Binding needs to be relative to the class, NOT top level
+  def secure_eval(namespace)
+    scoped = Scoped.new(%{timeout})
+    ts = create_timeseries(namespace.to_hash.delete "@timeseries")
+    ns = namespace.to_hash.merge(ts)
+    puts scoped.scoped_eval(ns)
+  end
+end
+
+json = <<-'EOF'
+%{namespace}
+EOF
+
+Wrapper.new.secure_eval(JSON.parse(json))

data/lib/rearview/templates/utilities.rb

@@ -0,0 +1,100 @@
+#
+# Add re-usable code/functions in this module
+#
+class Array
+  def mean
+    self.sum / self.length
+  end
+
+  def median
+    sorted = self.sort
+    mid    = self.length / 2
+    if self.length.odd?
+      sorted[mid].to_f
+    else
+      (sorted[mid-1] + sorted[mid]).to_f / 2.0
+    end
+  end
+
+  def sum
+    self.inject(0) { |total, n| total + n.to_f }
+  end
+
+  def percentile(number)
+    position = (number > 1) ? (number.to_f / 100) : number
+    arr = self.map { |x| x || 0 }
+    arr.sort[(arr.length * position) - 1]
+  end
+  
+  def sample_variance
+    return self.sum / (self.length - 1).to_f
+  end
+
+  def stdev
+    return Math.sqrt(self.sample_variance)
+  end
+end
+
+
+module MonitorUtilities
+  # checks if the value is outside the limits for all of the comparison values
+  def outside_limits?(value, comparison_values, limit_value, limit_type)
+    diffs = comparison_values.map { |v| value - v if value && v }.compact
+    diffs.length == 2 && ((limit_type == :lower && diffs.max < limit_value) || (limit_type == :upper && diffs.min > limit_value))
+  end
+
+  # percentage of minutes on this monitor that have values outside the limits
+  def percentage_errors(metric, comparison_metrics, limit_value, limit_type, minutes = @minutes)
+    raise "You can only define the limit type as :upper or :lower." if ![:upper, :lower].include? limit_type
+
+    zipped_values = metric.values.zip(*(Array(comparison_metrics).map(&:values)))
+
+    error_count = zipped_values.count do |(value, *comparison_values)|
+      outside_limits?(value, comparison_values, limit_value, limit_type)
+    end
+
+    (error_count.to_f / minutes.to_f) * 100
+  end
+
+  # checks for a deployment and if found returns data before and after the deploy along with the delta
+  def deploy_check(num_points, deploy, metric)
+    if metric == deploy
+      raise "You've passed the deploy metric to be analyzed against itself, which is not a valid analysis."
+    elsif metric.values.size < (num_points * 2) + 1
+      raise "Not enough data to evaluate. There must be #{num_points} data points before and after a deploy."
+    else
+      results = []
+      last_deploy = deploy.values.rindex { |v| !v.nil? }
+
+      if last_deploy
+        deploy_time = deploy.entries[last_deploy].timestamp
+
+        # If the num_points after the deploy is true then
+        if metric.entries.drop_while { |entry| entry.timestamp <= deploy_time }.length == num_points
+          before = metric.values.last((num_points * 2) + 1).first(num_points).sum
+          after  = metric.values.last(num_points).sum
+          delta  = before == 0 ? 0.0 : ((after - before) / before) * 100
+
+          results = [metric.label, before, after, delta]
+        end
+      end
+
+      results
+    end
+  end
+  
+  def form_error(metric, tolerance, actual)
+    "#{metric.label} experienced a standard deviation shift of #{actual.round(2)}, which is greater than the threshold of #{tolerance}."
+  end
+
+  def collect_comparisons(metric)
+    five_minute_sv = metric.values.each_slice(5).to_a.map { |pair| pair.stdev  }
+    five_minute_sv.each_slice(2).to_a.map { |pair| pair.sort }.map { |pair| pair[1] - pair[0] }
+  end
+
+  def collect_aberrations(*metrics, deviation)
+    metrics.map do |m|
+      collect_comparisons(m).inject([]) { |accum, shift| shift >= deviation ? form_error(m, deviation, shift) : "" }
+    end
+  end
+end # Class end

data/lib/rearview/uri_helper.rb

@@ -0,0 +1,7 @@
+module Rearview
+  class UriHelper
+    def self.rearview_uri(job)
+      "https://rearview.livingsocial.net/#dash/#{job.app_id}/expand/#{job.id}"
+    end
+  end
+end

data/lib/rearview/version.rb

@@ -0,0 +1,3 @@
+module Rearview
+  VERSION = "1.0.0"
+end

data/lib/rearview.rb

@@ -0,0 +1,75 @@
+require "rearview/engine"
+require "rearview/concerns"
+require "rearview/ext/state_machine"
+require "rearview/constants_module_maker"
+require 'rearview/logger'
+require 'rearview/cron_helper'
+require 'rearview/graphite_parser'
+require 'rearview/results_handler'
+require 'rearview/alerts_handler'
+require 'rearview/uri_helper'
+require 'rearview/alerts'
+require 'rearview/monitor_runner'
+require 'rearview/monitor_task'
+require 'rearview/distribute'
+require 'rearview/monitor_supervisor'
+require 'rearview/monitor_service'
+require 'rearview/configuration'
+require 'rearview/sandbox'
+require 'rearview/version'
+
+module Rearview
+  include Rearview::Logger
+
+  class << self
+    attr_accessor :monitor_service,:alert_clients
+  end
+
+  module_function
+
+  def logger
+    @logger ||= Rails.logger
+  end
+
+  def logger=(logger)
+    @logger = logger
+  end
+
+  def booted?
+    @booted
+  end
+
+  def configure
+    yield config
+  end
+
+  def config
+    @config ||= Rearview::Configuration.new
+  end
+
+  def boot!
+    @logger = config.logger if(config.logger.present?)
+    logger.info "[#{self}] booting..."
+    logger.info "[#{self}] using configuration: \n#{config}"
+    if config.verify?
+      logger.info "[#{self}] verifying sandbox..."
+      if Rearview::Sandbox.valid?
+        logger.info "[#{self}] sandbox verified"
+      else
+        logger.error "[#{self}] sandbox verification FAILED"
+      end
+    end
+    Celluloid.logger = @logger
+    jobs = ( config.preload_jobs? ? Job.schedulable : [] )
+    logger.info "[#{self}] starting up monitor service for (#{jobs.count}) job(s)"
+    @monitor_service = Rearview::MonitorService.new(jobs)
+    if config.monitor_enabled?
+      @monitor_service.startup
+    else
+      logger.warn "[#{self}] monitor disabled"
+    end
+    @alert_clients = Rearview::Alerts.registry.values
+    @booted = true
+  end
+
+end

data/lib/tasks/rearview_tasks.rake

@@ -0,0 +1,3 @@
+# Tasks made available to engine host..
+namespace :rearview do
+end

data/public/help/alert.html

@@ -0,0 +1,20 @@
+<table>
+    <tr>
+      <th colspan="2">Multiple alert URIs can be entered (separated by a space, comma or new line).</th>
+    <tr>
+        <th>URI</th>
+        <td>ALERT TYPE</td>
+    </tr>
+    <tr>
+        <th>mailto:first.last@hungrymachine.com</th>
+        <td>Send email alerts.</td>
+    </tr>
+    <tr>
+        <th>pagerduty:54232f6f4c4447efb6d15e20dbb7349b</th>
+        <td>Send alerts to pagerduty.</td>
+    </tr>
+    <tr>
+        <th>campfire://first.last@hungrymachine.com?token=yourtoken&amp;room=Test</th>
+        <td>Send alerts to a campfire room.</td>
+    </tr>
+</table>

data/public/help/quick.html

@@ -0,0 +1,34 @@
+<table>
+    <tr>
+        <th>@timeseries</th>
+        <td>Is the main collection of data retrieved from Graphite. The form is a json array of arrays of objects.</td>
+    </tr>
+    <tr>
+        <th>@a,@b,@c,...</th>
+        <td>Are convenience variables extracted from @timeline, by position. @a is the first entry, @b the second and so on. Note: These are not predictable when using wildcard metrics.</td>
+    </tr>
+    <tr>
+        <th>with_metrics { |a,b| .. }</th>
+        <td>Iterates over each metric in the timeseries aligning the timestamp for each metric across iterations. Block args are the variables you want to map to the metrics by position.</td>
+    </tr>
+    <tr>
+        <th>graph_value[label, timestamp, value]</th>
+        <td>Plots an x/y for the given label on the graph.</td>
+    </tr>
+    <tr>
+        <th>Array.mean</th>
+        <td>Calculates the mean of the Array of Numeric. e.g. @a.values.mean</td>
+    </tr>
+    <tr>
+        <th>Array.median</th>
+        <td>Calculates the median of the Array of Numeric. e.g. @a.values.median</td>
+    </tr>
+    <tr>
+        <th>Array.sum</th>
+        <td>Calculates the sum of the Array of Numeric. e.g. @a.values.sum</td>
+    </tr>
+    <tr>
+        <th>Array.percentile(n)</th>
+        <td>Calculates the n-th percentile of the Array of Numeric. e.g. @a.values.percentile(90)</td>
+    </tr>
+</table>

data/public/monitors/index.json

@@ -0,0 +1,3 @@
+[
+  { "name" : "Outage Monitor", "description" : "Monitor template to detect when an application is no longer sending metrics", "path" : "/monitors/outage.rb", "metrics" : ["foo.bar.baz"]}
+]

data/public/monitors/outage.rb

@@ -0,0 +1,2 @@
+# This template monitors a metric and if it detects nil (no data) for over a minute it alerts
+raise "Custom raise message goes here" if (@a.values.inject :+) == 0