RubyGems - triglav-agent - Versions diffs - 1.0.0.pre1 - Mend

triglav-agent 1.0.0.pre1

Files changed (68) hide show

checksums.yaml +7 -0
data/.gitignore +11 -0
data/.rspec +2 -0
data/.travis.yml +6 -0
data/.yardopts +6 -0
data/CHANGELOG.md +3 -0
data/CODE_OF_CONDUCT.md +49 -0
data/Gemfile +5 -0
data/LICENSE.txt +21 -0
data/README.md +59 -0
data/Rakefile +11 -0
data/bin/console +14 -0
data/bin/setup +8 -0
data/docs/Triglav/Agent/ApiClient/AuthenticationError.html +150 -0
data/docs/Triglav/Agent/ApiClient/ConnectionError.html +150 -0
data/docs/Triglav/Agent/ApiClient/Error.html +296 -0
data/docs/Triglav/Agent/ApiClient.html +545 -0
data/docs/Triglav/Agent/Base/CLI.html +498 -0
data/docs/Triglav/Agent/Base/Connection.html +258 -0
data/docs/Triglav/Agent/Base/Monitor.html +370 -0
data/docs/Triglav/Agent/Base/Processor.html +623 -0
data/docs/Triglav/Agent/Base/Setting.html +1081 -0
data/docs/Triglav/Agent/Base/Worker.html +635 -0
data/docs/Triglav/Agent/Base.html +121 -0
data/docs/Triglav/Agent/Configuration.html +967 -0
data/docs/Triglav/Agent/Error.html +130 -0
data/docs/Triglav/Agent/HashUtil.html +351 -0
data/docs/Triglav/Agent/LogFormatter.html +271 -0
data/docs/Triglav/Agent/Logger.html +287 -0
data/docs/Triglav/Agent/StorageFile.html +1130 -0
data/docs/Triglav/Agent/Timer.html +424 -0
data/docs/Triglav/Agent/TooManyError.html +134 -0
data/docs/Triglav/Agent.html +131 -0
data/docs/Triglav.html +119 -0
data/docs/_index.html +335 -0
data/docs/class_list.html +51 -0
data/docs/css/common.css +1 -0
data/docs/css/full_list.css +58 -0
data/docs/css/style.css +481 -0
data/docs/file.LICENSE.html +72 -0
data/docs/file.README.html +137 -0
data/docs/file_list.html +61 -0
data/docs/frames.html +17 -0
data/docs/index.html +137 -0
data/docs/js/app.js +243 -0
data/docs/js/full_list.js +216 -0
data/docs/js/jquery.js +4 -0
data/docs/method_list.html +659 -0
data/docs/top-level-namespace.html +112 -0
data/example/config.yml +40 -0
data/lib/triglav/agent/api_client.rb +199 -0
data/lib/triglav/agent/base/cli.rb +98 -0
data/lib/triglav/agent/base/connection.rb +30 -0
data/lib/triglav/agent/base/monitor.rb +32 -0
data/lib/triglav/agent/base/processor.rb +136 -0
data/lib/triglav/agent/base/setting.rb +112 -0
data/lib/triglav/agent/base/worker.rb +95 -0
data/lib/triglav/agent/configuration.rb +79 -0
data/lib/triglav/agent/error.rb +4 -0
data/lib/triglav/agent/hash_util.rb +36 -0
data/lib/triglav/agent/logger.rb +50 -0
data/lib/triglav/agent/storage_file.rb +144 -0
data/lib/triglav/agent/timer.rb +80 -0
data/lib/triglav/agent/version.rb +5 -0
data/lib/triglav/agent.rb +20 -0
data/lib/triglav-agent.rb +1 -0
data/triglav-agent.gemspec +33 -0
metadata +250 -0

data/docs/top-level-namespace.html ADDED Viewed

@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.5
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <iframe id="search_frame" src="class_list.html"></iframe>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Triglav.html" title="Triglav (module)">Triglav</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Thu Feb 23 23:20:33 2017 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.5 (ruby-2.3.2).
+</div>
+    </div>
+  </body>
+</html>

data/example/config.yml ADDED Viewed

@@ -0,0 +1,40 @@
+defaults: &defaults
+  serverengine:
+    log: 'STDOUT'
+    log_level: 'debug'
+    log_rotate_age: 5
+    log_rotate_size: 10485760
+  triglav:
+    url: <%= ENV['TRIGLAV_URL'] || 'http://localhost:7800' %>
+    credential:
+      username: <%= ENV['TRIGLAV_USERNAME'] || 'triglav_test' %>
+      password: <%= ENV['TRIGLAV_PASSWORD'] || 'triglav_test' %>
+      authenticator: local
+    timeout: 60
+    debugging: false
+    retries: 3
+    retry_interval: 3 # sec
+  vertica:
+    monitor_interval: 5
+    parallel_size: 1 # default: 1
+    parallel_type: 'thread' # thread or process. default: thread
+    connection_pool_size: 1 # default: same with parallel.size
+    connection_pool_timeout: 60 # sec. default: 60
+    date_column: d
+    timestamp_column: t
+    connection_info:
+      "vertica://":
+        host: <%= ENV['VERTICA_HOST'] %>
+        port: <%= ENV['VERTICA_PORT'] %>
+        database: <%= ENV['VERTICA_DATABASE'] %>
+        user: <%= ENV['VERTICA_USER'] %>
+        password: <%= ENV['VERTICA_PASSWORD'] %>
+        resource_pool: <%= ENV['VERTICA_RESOURCE_POOL'] %>
+        interruptable: true
+        read_timeout: 5
+development:
+  <<: *defaults
+test:
+  <<: *defaults

data/lib/triglav/agent/api_client.rb ADDED Viewed

@@ -0,0 +1,199 @@
+require 'triglav/agent/storage_file'
+require 'triglav_client'
+module Triglav::Agent
+  # This Triglav client connects to triglav API with $setting.triglav.url,
+  # and authenticates with $setting.triglav.credential, and
+  # stores the token into $setting.token_file.
+  #
+  # Re-authenticate automatically if token is expired
+  #
+  #     require 'triglav/agent/api_client'
+  #
+  #     api_client = Triglav::Agent::ApiClient.new
+  #     resources = api_client.list_resources(uri_prefix)
+  #     resources.each do |resource|
+  #       events = get_events(resource) # implement this!
+  #       api_client.send_messages(events)
+  #     end
+  class ApiClient
+    class Error < StandardError
+      attr_reader :cause
+      def initialize(message, cause)
+        @cause = cause
+        super(message)
+      end
+    end
+    class AuthenticationError < Error; end
+    class ConnectionError < Error; end
+    # Initialize TriglavClient
+    def initialize(opts = {})
+      @opts = opts || {}
+      config = TriglavClient::Configuration.new do |config|
+        uri = URI.parse(triglav_url)
+        config.scheme = uri.scheme
+        config.host = "#{uri.host}:#{uri.port}"
+        config.timeout = timeout if timeout
+        config.debugging = debugging if debugging
+      end
+      @api_client = TriglavClient::ApiClient.new(config)
+      initialize_current_token
+      authenticate
+    end
+    def close
+      # typhoeus makes a new connection and disconnects it on each request
+    end
+    # List resources required to be monitored
+    #
+    # @param [String] uri_prefix
+    # @return [Array of TriglavClient::ResourceEachResponse] array of resources
+    # @see TriglavClient::ResourceEachResponse
+    def list_aggregated_resources(uri_prefix)
+      $logger.debug { "ApiClient#list_aggregated_resources(#{uri_prefix.inspect})" }
+      resources_api = TriglavClient::ResourcesApi.new(@api_client)
+      handle_error { resources_api.list_aggregated_resources(uri_prefix) }
+    end
+    # Send messages
+    #
+    # @param [Array of TriglavClient::MessageRequest] array of event messages
+    # @see TriglavClient::MessageRequest
+    def send_messages(events)
+      $logger.debug { "ApiClient#send_messages(#{events.inspect})" }
+      messages_api = TriglavClient::MessagesApi.new(@api_client)
+      handle_error { messages_api.send_messages(events) }
+    end
+    def authorized?
+      @current_token.has_key?(:access_token)
+    end
+    private
+    # Authenticate
+    #
+    # 1. Another process saved a newer token onto the token_file => read it
+    # 2. The token saved on the token_file is same with current token => re-authenticate
+    # 3. The token saved on the token_file is older than the current token
+    #   => unknown situation, re-authenticate and save into token_file to refresh anyway
+    # 4. No token is saved on the token_file => authenticate
+    def authenticate
+      $logger.debug { 'ApiClient#authenticate' }
+      StorageFile.open(token_file) do |fp|
+        token = fp.load
+        if should_read_token_from_file?(token)
+          $logger.debug { "Read token from file" }
+          update_current_token(token)
+          return
+        end
+        $logger.debug { "AuthApi#create_token" }
+        auth_api = TriglavClient::AuthApi.new(@api_client)
+        credential = TriglavClient::Credential.new(
+          username: username, password: password, authenticator: authenticator
+        )
+        handle_auth_error do
+          $logger.debug { 'TriglavClient::AuthApi' }
+          result = auth_api.create_token(credential)
+          token = {access_token: result.access_token}
+          update_current_token(token)
+          fp.dump(token)
+        end
+      end
+    end
+    def initialize_current_token
+      @current_token = {
+        access_token: (@api_client.config.api_key['Authorization'] = String.new),
+      }
+    end
+    def update_current_token(token)
+      @current_token[:access_token].replace(token[:access_token])
+    end
+    def should_read_token_from_file?(token)
+      return true if @current_token[:access_token].empty? && !(token[:access_token] ||= '').empty?
+      return @current_token[:access_token] != token[:access_token]
+    end
+    def handle_auth_error(&block)
+      retries = 0
+      begin
+        yield
+      rescue TriglavClient::ApiError => e
+        if e.code == 0
+          if retries < max_retries
+            sleep retry_interval
+            retries += 1
+            retry
+          end
+          raise ConnectionError.new("Could not connect to #{triglav_url} with #{retries} retries", e)
+        elsif e.message == 'Unauthorized'.freeze
+          raise AuthenticationError.new("Failed to authenticate on triglav API.".freeze, e)
+        else
+          raise Error.new(e.message, e)
+        end
+      end
+    end
+    def handle_error(&block)
+      retries = 0
+      begin
+        yield
+      rescue TriglavClient::ApiError => e
+        if e.code == 0
+          if retries < max_retries
+            sleep retry_interval
+            retries += 1
+            retry
+          end
+          raise ConnectionError.new("Could not connect to #{triglav_url} with #{retries} retries", e)
+        elsif e.message == 'Unauthorized'.freeze
+          authenticate
+          retry
+        else
+          raise Error.new(e.message, e)
+        end
+      end
+    end
+    def triglav_url
+      @opts[:url] || $setting.dig(:triglav, :url)
+    end
+    def username
+      @opts.dig(:credential, :username) || $setting.dig(:triglav, :credential, :username)
+    end
+    def password
+      @opts.dig(:credential, :password) || $setting.dig(:triglav, :credential, :password)
+    end
+    def authenticator
+      @opts.dig(:credential, :authenticator) || $setting.dig(:triglav, :credential, :authenticator)
+    end
+    def timeout
+      @opts[:timeout] || $setting.dig(:triglav, :timeout)
+    end
+    def debugging
+      @opts[:debugging] || $setting.dig(:triglav, :debugging)
+    end
+    def max_retries
+      @opts[:retries] || $setting.dig(:triglav, :retries) || 3
+    end
+    def retry_interval
+      @opts[:retry_interval] || $setting.dig(:triglav, :retry_interval) || 3 # second
+    end
+    def token_file
+      @opts[:token_file] || $setting.token_file
+    end
+  end
+end

data/lib/triglav/agent/base/cli.rb ADDED Viewed

@@ -0,0 +1,98 @@
+require "triglav/agent/configuration"
+require "triglav/agent/base/setting"
+require 'optparse'
+require 'serverengine'
+module Triglav::Agent
+  module Base
+    # A base class for cli option parser
+    #
+    # This base class usually should be enough for agent plugins, but
+    # you can override this class and configure with Configuration#cli_classe=
+    class CLI
+      def run
+        opts, _ = parse_options(ARGV)
+        $setting = Configuration.setting_class.new(opts)
+        $logger = $setting.logger
+        se = ServerEngine.create(nil, Configuration.worker_module) do
+          $setting.serverengine_options
+        end
+        se.run
+      end
+      def default_opts
+        {
+          config: 'config.yml',
+          dotenv: false,
+          debug: false,
+        }
+      end
+      def option_parser(opts = {})
+        op = OptionParser.new
+        self.class.module_eval do
+          define_method(:usage) do |msg = nil|
+            puts op.to_s
+            puts "error: #{msg}" if msg
+            exit 1
+          end
+        end
+        op.on('-c', '--config VALUE', "Config file (default: #{opts[:config]})") {|v|
+          opts[:config] = v
+        }
+        op.on('-s', '--status VALUE', "Status stroage file (default: status.yml)") {|v|
+          opts[:status] = v
+        }
+        op.on('-t', '--token VALUE', "Triglav access token storage file (default: token.yml)") {|v|
+          opts[:token] = v
+        }
+        op.on('--dotenv', "Load environment variables from .env file (default: #{opts[:dotenv]})") {|v|
+          opts[:dotenv] = v
+        }
+        op.on('--debug', "Debug mode (default: #{opts[:debug]})") {|v|
+          opts[:debug] = v
+        }
+        op.on('-h', '--help', "help") {|v|
+          opts[:help] = v
+        }
+        # serverengine options
+        op.on('--log VALUE', "Log path (default: #{Setting::DEFAULT_LOG})") {|v|
+          opts[:log] = v
+        }
+        op.on('--log-level VALUE', "Log level (default: #{Setting::DEFAULT_LOG_LEVEL})") {|v|
+          opts[:log_level] = v
+        }
+        op.banner += ''
+        op
+      end
+      def parse_options(argv = ARGV)
+        opts = default_opts
+        op = option_parser(opts)
+        begin
+          args = op.parse(argv)
+        rescue OptionParser::InvalidOption => e
+          usage e.message
+        end
+        if opts[:help]
+          usage
+        end
+        if opts[:config].nil?
+          usage "--config VALUE is required"
+        end
+        if !File.readable?(opts[:config])
+          usage "Config file '#{opts[:config]}' does not exist or not readable"
+        end
+        [opts, args]
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/connection.rb ADDED Viewed

@@ -0,0 +1,30 @@
+module Triglav::Agent
+  module Base
+    # An abstract base class of Connection.
+    #
+    # Wrap a connection to a storage.
+    # You can implement any methods which you want to use in your Monitor class.
+    #
+    # You have to implement following methods:
+    #
+    # * initialize
+    # * close
+    #
+    # An instance is created for each `resource_uri_prefix`, that is,
+    # shared among resources with of same `resource_uri_prefix`.
+    #
+    # Note that multiple connection instances would be created
+    # when multiple parallel threads are created in Processor class.
+    class Connection
+      # @param [Hash] connection_info
+      def initialize(connection_info)
+        raise NotImplementedError
+      end
+      # close the connection
+      def close
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/monitor.rb ADDED Viewed

@@ -0,0 +1,32 @@
+module Triglav::Agent
+  module Base
+    # An abstract class of Monitor.
+    #
+    # Monitor your storage and send messages to triglav.
+    #
+    # You have to implement following methods:
+    #
+    # * initialize
+    # * process
+    #
+    # An instance is created per a `resource`.
+    # Connection is shared among same `resource_uri_prefix`.
+    #
+    # Note that multiple instances would be created,
+    # one instance for one parallel thread basically, and
+    # `#process` is ran concurrently.
+    class Monitor
+      # @param [Triglav::Agent::Base::Connection] connection
+      # @param [TriglavClient::ResourceResponse] resource
+      def initialize(connection, resource)
+        raise NotImplementedError
+      end
+      # @yield [events] Gives an array of events
+      def process(&block)
+        raise NotImplementedError
+        # yield(events) if block_given?
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/processor.rb ADDED Viewed

@@ -0,0 +1,136 @@
+require 'parallel'
+require 'connection_pool'
+module Triglav::Agent
+  module Base
+    # Triglav agent processor class.
+    #
+    # An instance is created for a `resource_uri_prefix`.
+    #
+    # You usually do not need to customize this class, but if you want to
+    # implement your original, configure
+    #
+    #     Triglav::Agent::Configuration.processor_class
+    class Processor
+      attr_reader :worker, :resource_uri_prefix
+      def initialize(worker, resource_uri_prefix)
+        @worker = worker
+        @resource_uri_prefix = resource_uri_prefix
+      end
+      def self.max_consecuitive_error_count
+        3
+      end
+      def process
+        before_process
+        success_count = 0
+        consecutive_error_count = 0
+        Parallel.each(resources, parallel_opts) do |resource|
+          raise Parallel::Break if stopped?
+          events = nil
+          begin
+            @connection_pool.with do |connection|
+              monitor = monitor_class.new(connection, resource)
+              monitor.process do |_events|
+                events = _events
+                $logger.info { "send_messages:#{events.map(&:to_hash).to_json}" }
+                @api_client_pool.with {|api_client| api_client.send_messages(events) }
+              end
+            end
+            @mutex.synchronize do
+              success_count += 1
+              consecutive_error_count = 0
+            end
+          rescue => e
+            log_error(e)
+            $logger.info { "failed_events:#{events.map(&:to_hash).to_json}" } if events
+            @mutex.synchronize do
+              raise TooManyError if (consecutive_error_count += 1) > self.class.max_consecuitive_error_count
+            end
+          end
+        end
+        success_count
+      ensure
+        after_process
+      end
+      def total_count
+        resources.size
+      end
+      private
+      def before_process
+        @connection_pool = ConnectionPool.new(connection_pool_opts) {
+          connection_class.new(get_connection_info(resource_uri_prefix))
+        }
+        @api_client_pool = ConnectionPool.new(connection_pool_opts) {
+          ApiClient.new # renew connection
+        }
+        @mutex = Mutex.new
+      end
+      def after_process
+        @connection_pool.shutdown {|conn| conn.close rescue nil }
+        @api_client_pool.shutdown {|conn| conn.close rescue nil }
+      end
+      def connection_class
+        Configuration.connection_class
+      end
+      def monitor_class
+        Configuration.monitor_class
+      end
+      def name
+        Configuration.name
+      end
+      def resources
+        return @resources if @resources
+        @resources = ApiClient.new.list_aggregated_resources(resource_uri_prefix) || []
+        $logger.debug { "resource_uri_prefix:#{resource_uri_prefix} resources.size:#{@resources.size}" }
+        @resources
+      end
+      def parallel_opts
+        parallel_type == 'process' ? {in_processes: parallel_size} : {in_threads: parallel_size}
+      end
+      def connection_pool_opts
+        {size: connection_pool_size, timeout: connection_pool_timeout}
+      end
+      def parallel_size
+        $setting.dig(name, :parallel_size) || 1
+      end
+      def parallel_type
+        $setting.dig(name, :parallel_type) || 'thread'
+      end
+      def connection_pool_size
+        $setting.dig(name, :connection_pool_size) || parallel_size
+      end
+      def connection_pool_timeout
+        $setting.dig(name, :connection_pool_timeout) || 60
+      end
+      def get_connection_info(resource_uri_prefix)
+        $setting.dig(name, :connection_info)[resource_uri_prefix]
+      end
+      def log_error(e)
+        $logger.error { "#{e.class} #{e.message} #{e.backtrace.join("\\n")}" } # one line
+      end
+      def stopped?
+        worker.stopped? if worker and worker.respond_to?(:stopped?)
+      end
+    end
+  end
+end