netuitived 0.10.1 → 1.0.1

data/README.md

@@ -1,20 +1,31 @@
-# netuitived
-a druby application that allows for the submission and aggregation of metrics through the use of DRbObjects
+NetuitiveD
+===========
 
-How to install using gem:
+NetuitiveD is a dRuby application submits and aggregates metrics through the use of DRbObjects. NetuitiveD is meant to work in conjunction with the [netuitive_ruby_api](https://rubygems.org/gems/netuitive_ruby_api) and [netuitive_rails_agent](https://rubygems.org/gems/netuitive_rails_agent) gems to help [Netuitive](https://www.netuitive.com) monitor your Ruby applications.
 
-run the command:
+For more information on NetuitiveD, see our Ruby agent [help docs](https://help.netuitive.com/Content/Misc/Datasources/new_ruby_datasource.htm), or contact Netuitive support at [support@netuitive.com](mailto:support@netuitive.com).
+
+Installing and Running NetuitiveD
+---------------------------------
+
+### Install
 
      gem install netuitived
 
-How to run:
+### Run
 
-Once the gem has been installed run the start script (make sure the user it's being run as has access rights to the gem's install directory):
+1. Run the start script (make sure the user using the command has access rights to the 
+gem's install directory):
 
-	netuitived start
+	   netuitived start
 
-The start script will prompt you for the API key (which is found from generating a datasource in the Netuitive ui) and the element name (if unsure what to put, just put the name of your ruby application)
+1. The start script will prompt you for an API key and an element name. You can get an API key from creating a Ruby datasource in [Netuitive](https://app.netuitive.com/auth/login).
 
-That's it, you're up and running. The extra config information is found in config/agent.yml. Each field in the config file can also be set using environment variables. 
+        please enter an element name:
+        my_app_name
+        please enter an api key:
+        DEMOab681D46bf5616dba8337c85DEMO
+        
+    If you're unsure about what element name you should use, just use the name of your Ruby application.
 
-This daemon is meant to be used in conjunction with the netuitive_ruby_api (https://rubygems.org/gems/netuitive_ruby_api) and netuitive_rails_agent (https://rubygems.org/gems/netuitive_rails_agent) gems.
+>**Note:** The extra config information is found in config/agent.yml. Each field in the config file can also be set using environment variables. 

data/bin/netuitived

@@ -1,57 +1,16 @@
 #!/usr/bin/env ruby
+require 'drb/drb'
+require 'netuitived'
 require 'yaml'
-gem_root= File.expand_path("../..", __FILE__)
-data=YAML.load_file "#{gem_root}/config/agent.yml"
-if ARGV[0] == "start"
-	written = false
-	elementName=ENV["NETUITIVED_ELEMENT_NAME"]
-	if(elementName == nil or elementName == "")
-		elementName = data["elementName"]
-		if(elementName == "elementName" or elementName == "")
-			puts "please enter an element name: "
-			elementName = STDIN.gets.chomp
-			data["elementName"] = elementName
-			written = true
-		end
-	end
-	apiId=ENV["NETUITIVED_API_ID"]
-	if(apiId == nil or apiId == "")
-		apiId = data["apiId"]
-		if(apiId == "apiId" or apiId == "")
-			puts "please enter an api key: "
-			apiId = STDIN.gets.chomp
-			data["apiId"] = apiId
-			written = true
-		end
-	end
-	if written
-		File.open("#{gem_root}/config/agent.yml", 'w') {|f| f.write data.to_yaml } 
-	end
-	require 'drb/drb'
-	netuitivedAddr=data["netuitivedAddr"]
-	netuitivedPort=data["netuitivedPort"]
-	SERVER_URI="druby://#{netuitivedAddr}:#{netuitivedPort}"
-	DRb.start_service
-	begin
-		DRbObject.new_with_uri(SERVER_URI).stopServer
-		sleep(2)
-	rescue
-		
-	end
-    require 'netuitived'
-    puts "netuitived started"
-elsif ARGV[0] == "stop"
-	require 'drb/drb'
-	netuitivedAddr=data["netuitivedAddr"]
-	netuitivedPort=data["netuitivedPort"]
-	SERVER_URI="druby://#{netuitivedAddr}:#{netuitivedPort}"
-	DRb.start_service
-	begin
-		DRbObject.new_with_uri(SERVER_URI).stopServer
-		puts "netuitived stopped"
-	rescue
-		puts "netuitived isn't running"
-	end
+
+# Check command line arguments
+case ARGV[0]
+when 'run' # runs netuitived in the foreground
+  Netuitived.start(true)
+when 'start' # runs netuitived in the background
+  Netuitived.start
+when 'stop' # stops running netuitived
+  Netuitived.stop
 else
-    puts "invalid option. options are: start, stop"
+  puts 'invalid option. options are: run, start, stop'
 end

data/config/agent.yml

@@ -1,4 +1,7 @@
 #all are configurable using environment variables
+logLocation: #absolute path of log file. leave blank for default location in the gem directory. environment variable: NETUITIVED_LOG_LOCATION
+logAge: daily #Number of old log files to keep, or frequency of rotation (daily, weekly or monthly). environment variable: NETUITIVED_LOG_AGE
+logSize: #Maximum logfile size (only applies when shift_age is a number). environment variable: NETUITIVED_LOG_SIZE
 apiId: apiId #apiId for the datasources. found in the datasources section in the netuitive ui. environment variable: NETUITIVED_API_ID
 baseAddr: "api.app.netuitive.com" #where to send the data. in most cases use the default. environment variable: NETUITIVED_BASE_ADDR
 port: nil #in most cases this will be nil. environment variable: NETUITIVED_PORT

data/lib/netuitived.rb

@@ -1,12 +1,137 @@
-fork do
-	require 'netuitive/netuitived_config_manager'
-	require 'netuitive/scheduler'
-	require 'drb/drb'
-	require 'netuitive/netuitived_server'
-	ConfigManager::setup
-	Scheduler::startSchedule
-	NETUITIVE_URI="druby://#{ConfigManager.netuitivedAddr}:#{ConfigManager.netuitivedPort}"
-	FRONT_OBJECT=NetuitivedServer.new 
-	DRb.start_service(NETUITIVE_URI, FRONT_OBJECT)
-	DRb.thread.join
-end
+require 'drb/drb'
+require 'net/http'
+require 'json'
+require 'yaml'
+require 'drb/drb'
+require 'logger'
+require 'netuitived/config_manager'
+require 'netuitived/netuitive_logger'
+require 'netuitived/ingest_event'
+require 'netuitived/ingest_tag'
+require 'netuitived/ingest_metric'
+require 'netuitived/ingest_sample'
+require 'netuitived/ingest_element'
+require 'netuitived/netuitived_server'
+require 'netuitived/api_emissary'
+require 'netuitived/metric_aggregator'
+require 'netuitived/event_handler'
+require 'netuitived/scheduler'
+
+##
+# Provides facilities for running Netuitived
+class Netuitived
+  class << self
+    attr_reader :server_uri
+
+    ##
+    # The DRb front object
+    #
+    # @return [NetuitivedServer] The NetuitivedServer front object
+    def front_object
+      @front_object ||= new_front_object
+    end
+
+    def new_front_object
+      apiEmissary = NetuitiveD::APIEmissary.new
+      NetuitiveD::NetuitivedServer.new(NetuitiveD::MetricAggregator.new(apiEmissary), NetuitiveD::EventHandler.new(apiEmissary))
+    end
+
+    ##
+    # Checks that config values are provided and prompts for them if not
+    def interactively_check_config
+      gem_root = File.expand_path('../..', __FILE__)
+      data = YAML.load_file("#{gem_root}/config/agent.yml")
+
+      written = false
+      element_name = ENV['NETUITIVED_ELEMENT_NAME'] || data['elementName'] || ''
+
+      if element_name == 'elementName' || element_name == ''
+        puts 'please enter an element name: '
+        element_name = STDIN.gets.chomp
+        data['elementName'] = element_name
+        written = true
+      end
+
+      api_id = ENV['NETUITIVED_API_ID'] || data['apiId'] || ''
+
+      if api_id == 'apiId' || api_id == ''
+        puts 'please enter an api key: '
+        api_id = STDIN.gets.chomp
+        data['apiId'] = api_id
+        written = true
+      end
+
+      return unless written
+
+      File.open("#{gem_root}/config/agent.yml", 'w') { |f| f.write data.to_yaml }
+    end
+
+    ##
+    # Starts netuitived
+    #
+    # @param foreground [true, false] If true, run netuitived in the foreground
+    def start(foreground = false)
+      # Maintain the initial setup behavior
+      interactively_check_config
+
+      # Load the config from disk into ConfigManager
+      load_config
+
+      # Stop the service if it's already running
+      stop(true)
+
+      # Create a proc to run netuitived
+      runner = proc do
+        NetuitiveD::NetuitiveLogger.log.debug 'starting scheduler'
+        NetuitiveD::Scheduler.startSchedule
+        NetuitiveD::NetuitiveLogger.log.debug 'starting drb service'
+        begin
+          DRb.start_service(server_uri, front_object)
+          DRb.thread.join
+        rescue => e
+          NetuitiveD::NetuitiveLogger.log.error "drb error: #{e.message} backtrace: #{e.backtrace}"
+        end
+      end
+
+      if foreground
+        puts 'netuitived running'
+        runner.call
+      else
+        fork(&runner)
+        puts 'netuitived started'
+      end
+    end
+
+    ##
+    # Stops netuitived if it's running
+    #
+    # @param suppress_not_running_message [true, false] suppresses the "netuitived isn't running" message
+    def stop(suppress_not_running_message = false)
+      load_config
+
+      DRb.start_service
+
+      begin
+        DRbObject.new_with_uri(@server_uri).stopServer
+        puts 'netuitived stopped'
+      rescue
+        puts "netuitived isn't running" unless suppress_not_running_message
+      end
+    end
+
+    private
+
+    ##
+    # Loads the configuration if necessary
+    def load_config
+      unless @config_manager_setup
+        NetuitiveD::ConfigManager.load_config
+        NetuitiveD::NetuitiveLogger.setup
+        NetuitiveD::ConfigManager.read_config
+      end
+
+      @config_manager_setup = true
+      @server_uri ||= "druby://#{NetuitiveD::ConfigManager.netuitivedAddr}:#{NetuitiveD::ConfigManager.netuitivedPort}".freeze
+    end
+  end
+end

data/lib/netuitived/api_emissary.rb

@@ -0,0 +1,51 @@
+module NetuitiveD
+  class APIEmissary
+    def sendElements(elementString)
+      NetuitiveD::NetuitiveLogger.log.debug 'start sending elements'
+      send("/ingest/ruby/#{NetuitiveD::ConfigManager.apiId}", elementString)
+      NetuitiveD::NetuitiveLogger.log.debug 'finish sending elements'
+    end
+
+    def sendEvents(eventString)
+      NetuitiveD::NetuitiveLogger.log.debug 'finish sending events'
+      send("/ingest/events/ruby/#{NetuitiveD::ConfigManager.apiId}", eventString)
+      NetuitiveD::NetuitiveLogger.log.debug 'finish sending events'
+    end
+
+    def send(uri, body)
+      NetuitiveD::NetuitiveLogger.log.debug "post body: #{body}"
+      req = Net::HTTP::Post.new(uri.to_s, 'Content-Type' => 'application/json')
+      req.body = body
+      NetuitiveD::NetuitiveLogger.log.debug 'starting post'
+      port = if NetuitiveD::ConfigManager.port =~ /(.*)nil(.*)/
+               nil
+             else
+               NetuitiveD::ConfigManager.port.to_int
+             end
+      NetuitiveD::NetuitiveLogger.log.debug "port: #{port}"
+      NetuitiveD::NetuitiveLogger.log.debug "path: #{req.path}"
+      NetuitiveD::NetuitiveLogger.log.debug "addr: #{NetuitiveD::ConfigManager.baseAddr}"
+      response = Net::HTTP.start(NetuitiveD::ConfigManager.baseAddr.to_s, port, use_ssl: true, read_timeout: 30, open_timeout: 30) do |http|
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        http.ssl_version = :SSLv3
+        http.request req
+      end
+      NetuitiveD::NetuitiveLogger.log.debug 'post finished'
+      if response.code.to_s != '202' && response.code.to_s != '200'
+        NetuitiveD::NetuitiveLogger.log.error "Response from submitting netuitive metrics to api
+        code: #{response.code}
+        message: #{response.message}
+        body: #{response.body}"
+      else
+        NetuitiveD::NetuitiveLogger.log.info "Response from submitting netuitive metrics to api
+        code: #{response.code}
+        message: #{response.message}
+        body: #{response.body}"
+      end
+    rescue => exception
+      NetuitiveD::NetuitiveLogger.log.error 'error with http post: '
+      NetuitiveD::NetuitiveLogger.log.error exception.message
+      NetuitiveD::NetuitiveLogger.log.error exception.backtrace
+    end
+  end
+end

data/lib/netuitived/config_manager.rb

@@ -0,0 +1,91 @@
+module NetuitiveD
+  class ConfigManager
+    class << self
+      attr_reader :apiId
+
+      attr_reader :baseAddr
+
+      attr_reader :port
+
+      attr_reader :elementName
+
+      attr_reader :netuitivedAddr
+
+      attr_reader :netuitivedPort
+
+      attr_reader :interval
+
+      attr_reader :data
+
+      def property(name, var, default = nil)
+        prop = ENV[var]
+        prop = data[name] if prop.nil? || (prop == '')
+        return prop unless prop.nil? || (prop == '')
+        default
+      end
+
+      def boolean_property(name, var)
+        prop = ENV[var].nil? ? nil : ENV[var].dup
+        if prop.nil? || (prop == '')
+          prop = data[name]
+        else
+          prop.strip!
+          prop = prop.casecmp('true').zero?
+        end
+        prop
+      end
+
+      def float_property(name, var)
+        prop = ENV[var].nil? ? nil : ENV[var]
+        if prop.nil? || (prop == '')
+          data[name].to_f
+        else
+          prop.to_f
+        end
+      end
+
+      def string_list_property(name, var)
+        list = []
+        prop = ENV[var].nil? ? nil : ENV[var].dup
+        if prop.nil? || (prop == '')
+          list = data[name] if !data[name].nil? && data[name].is_a?(Array)
+        else
+          list = prop.split(',')
+        end
+        list.each(&:strip!) unless list.empty?
+        list
+      end
+
+      def load_config
+        gem_root = File.expand_path('../../..', __FILE__)
+        @data = YAML.load_file "#{gem_root}/config/agent.yml"
+      end
+
+      def read_config
+        @apiId = property('apiId', 'NETUITIVED_API_ID')
+        @baseAddr = property('baseAddr', 'NETUITIVED_BASE_ADDR')
+        @port = property('port', 'NETUITIVED_PORT')
+        @elementName = property('elementName', 'NETUITIVED_ELEMENT_NAME')
+        @netuitivedAddr = property('netuitivedAddr', 'NETUITIVED_NETUITIVED_ADDR')
+        @netuitivedPort = property('netuitivedPort', 'NETUITIVED_NETUITIVED_PORT')
+        @interval = property('interval', 'NETUITIVED_INTERVAL')
+        debugLevelString = property('debugLevel', 'NETUITIVED_DEBUG_LEVEL')
+        NetuitiveD::NetuitiveLogger.log.level = if debugLevelString == 'error'
+                                                  Logger::ERROR
+                                                elsif debugLevelString == 'info'
+                                                  Logger::INFO
+                                                elsif debugLevelString == 'debug'
+                                                  Logger::DEBUG
+                                                else
+                                                  Logger::ERROR
+                                                end
+        NetuitiveD::NetuitiveLogger.log.debug "read config file. Results:
+          apiId: #{apiId}
+          baseAddr: #{baseAddr}
+          port: #{port}
+          elementName: #{elementName}
+          debugLevel: #{debugLevelString}"
+      end
+    end
+  end
+end

data/lib/netuitived/event_handler.rb

@@ -0,0 +1,40 @@
+module NetuitiveD
+  class EventHandler
+    def initialize(apiEmissary)
+      @apiEmissary = apiEmissary
+    end
+
+    def handleEvent(message, timestamp, title, level, source, type, tags)
+      NetuitiveD::NetuitiveLogger.log.debug "self: #{object_id}"
+      NetuitiveD::NetuitiveLogger.log.debug "Thread: #{Thread.current.object_id}"
+      NetuitiveD::NetuitiveLogger.log.debug "Received event: message:#{message}, timestamp:#{timestamp}, title:#{title}, level:#{level}, source:#{source}, type:#{type}, tags:#{tags}"
+      event = NetuitiveD::IngestEvent.new(NetuitiveD::ConfigManager.elementName, message, timestamp, title, level, source, type, tags)
+      events = [event]
+      eventString = events.to_json
+      @apiEmissary.sendEvents(eventString)
+    rescue => e
+      NetuitiveD::NetuitiveLogger.log.error "error in handling event: #{e.message}, backtrace: #{e.backtrace}"
+    end
+
+    def handleExceptionEvent(exception, klass, tags = {})
+      message = "Exception Message: #{exception[:message]}\n" if (defined? exception[:message]) && !exception[:message].nil?
+      message ||= ''
+      timestamp = Time.new
+      title = 'Ruby Exception'
+      level = 'Warning'
+      source = 'Ruby Agent'
+      type = 'INFO'
+      ingest_tags = []
+      tags [:Exception] = klass unless klass.nil?
+      tags.each do |key, value|
+        next if !(defined? value) || value.nil? || value == ''
+        ingest_tags << NetuitiveD::IngestTag.new(key, value)
+        message += "#{key}: #{value}\n"
+      end
+      message += "Backtrace:\n\t#{exception[:backtrace]}" if (defined? exception[:backtrace]) && !exception[:backtrace].nil?
+      handleEvent(message, timestamp, title, level, source, type, ingest_tags)
+    rescue => e
+      NetuitiveD::NetuitiveLogger.log.error "error in exception event: #{e.message}, backtrace: #{e.backtrace}"
+    end
+  end
+end