RubyGems - triglav-agent - Versions diffs - 1.0.0.pre1 - Mend

triglav-agent 1.0.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (68) hide show

checksums.yaml +7 -0
data/.gitignore +11 -0
data/.rspec +2 -0
data/.travis.yml +6 -0
data/.yardopts +6 -0
data/CHANGELOG.md +3 -0
data/CODE_OF_CONDUCT.md +49 -0
data/Gemfile +5 -0
data/LICENSE.txt +21 -0
data/README.md +59 -0
data/Rakefile +11 -0
data/bin/console +14 -0
data/bin/setup +8 -0
data/docs/Triglav/Agent/ApiClient/AuthenticationError.html +150 -0
data/docs/Triglav/Agent/ApiClient/ConnectionError.html +150 -0
data/docs/Triglav/Agent/ApiClient/Error.html +296 -0
data/docs/Triglav/Agent/ApiClient.html +545 -0
data/docs/Triglav/Agent/Base/CLI.html +498 -0
data/docs/Triglav/Agent/Base/Connection.html +258 -0
data/docs/Triglav/Agent/Base/Monitor.html +370 -0
data/docs/Triglav/Agent/Base/Processor.html +623 -0
data/docs/Triglav/Agent/Base/Setting.html +1081 -0
data/docs/Triglav/Agent/Base/Worker.html +635 -0
data/docs/Triglav/Agent/Base.html +121 -0
data/docs/Triglav/Agent/Configuration.html +967 -0
data/docs/Triglav/Agent/Error.html +130 -0
data/docs/Triglav/Agent/HashUtil.html +351 -0
data/docs/Triglav/Agent/LogFormatter.html +271 -0
data/docs/Triglav/Agent/Logger.html +287 -0
data/docs/Triglav/Agent/StorageFile.html +1130 -0
data/docs/Triglav/Agent/Timer.html +424 -0
data/docs/Triglav/Agent/TooManyError.html +134 -0
data/docs/Triglav/Agent.html +131 -0
data/docs/Triglav.html +119 -0
data/docs/_index.html +335 -0
data/docs/class_list.html +51 -0
data/docs/css/common.css +1 -0
data/docs/css/full_list.css +58 -0
data/docs/css/style.css +481 -0
data/docs/file.LICENSE.html +72 -0
data/docs/file.README.html +137 -0
data/docs/file_list.html +61 -0
data/docs/frames.html +17 -0
data/docs/index.html +137 -0
data/docs/js/app.js +243 -0
data/docs/js/full_list.js +216 -0
data/docs/js/jquery.js +4 -0
data/docs/method_list.html +659 -0
data/docs/top-level-namespace.html +112 -0
data/example/config.yml +40 -0
data/lib/triglav/agent/api_client.rb +199 -0
data/lib/triglav/agent/base/cli.rb +98 -0
data/lib/triglav/agent/base/connection.rb +30 -0
data/lib/triglav/agent/base/monitor.rb +32 -0
data/lib/triglav/agent/base/processor.rb +136 -0
data/lib/triglav/agent/base/setting.rb +112 -0
data/lib/triglav/agent/base/worker.rb +95 -0
data/lib/triglav/agent/configuration.rb +79 -0
data/lib/triglav/agent/error.rb +4 -0
data/lib/triglav/agent/hash_util.rb +36 -0
data/lib/triglav/agent/logger.rb +50 -0
data/lib/triglav/agent/storage_file.rb +144 -0
data/lib/triglav/agent/timer.rb +80 -0
data/lib/triglav/agent/version.rb +5 -0
data/lib/triglav/agent.rb +20 -0
data/lib/triglav-agent.rb +1 -0
data/triglav-agent.gemspec +33 -0
metadata +250 -0

data/docs/top-level-namespace.html ADDED Viewed

@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.5
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <iframe id="search_frame" src="class_list.html"></iframe>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Triglav.html" title="Triglav (module)">Triglav</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Thu Feb 23 23:20:33 2017 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.5 (ruby-2.3.2).
+</div>
+    </div>
+  </body>
+</html>

data/example/config.yml ADDED Viewed

@@ -0,0 +1,40 @@
+defaults: &defaults
+  serverengine:
+    log: 'STDOUT'
+    log_level: 'debug'
+    log_rotate_age: 5
+    log_rotate_size: 10485760
+  triglav:
+    url: <%= ENV['TRIGLAV_URL'] || 'http://localhost:7800' %>
+    credential:
+      username: <%= ENV['TRIGLAV_USERNAME'] || 'triglav_test' %>
+      password: <%= ENV['TRIGLAV_PASSWORD'] || 'triglav_test' %>
+      authenticator: local
+    timeout: 60
+    debugging: false
+    retries: 3
+    retry_interval: 3 # sec
+  vertica:
+    monitor_interval: 5
+    parallel_size: 1 # default: 1
+    parallel_type: 'thread' # thread or process. default: thread
+    connection_pool_size: 1 # default: same with parallel.size
+    connection_pool_timeout: 60 # sec. default: 60
+    date_column: d
+    timestamp_column: t
+    connection_info:
+      "vertica://":
+        host: <%= ENV['VERTICA_HOST'] %>
+        port: <%= ENV['VERTICA_PORT'] %>
+        database: <%= ENV['VERTICA_DATABASE'] %>
+        user: <%= ENV['VERTICA_USER'] %>
+        password: <%= ENV['VERTICA_PASSWORD'] %>
+        resource_pool: <%= ENV['VERTICA_RESOURCE_POOL'] %>
+        interruptable: true
+        read_timeout: 5
+development:
+  <<: *defaults
+test:
+  <<: *defaults

data/lib/triglav/agent/api_client.rb ADDED Viewed

@@ -0,0 +1,199 @@
+require 'triglav/agent/storage_file'
+require 'triglav_client'
+module Triglav::Agent
+  # This Triglav client connects to triglav API with $setting.triglav.url,
+  # and authenticates with $setting.triglav.credential, and
+  # stores the token into $setting.token_file.
+  #
+  # Re-authenticate automatically if token is expired
+  #
+  #     require 'triglav/agent/api_client'
+  #
+  #     api_client = Triglav::Agent::ApiClient.new
+  #     resources = api_client.list_resources(uri_prefix)
+  #     resources.each do |resource|
+  #       events = get_events(resource) # implement this!
+  #       api_client.send_messages(events)
+  #     end
+  class ApiClient
+    class Error < StandardError
+      attr_reader :cause
+      def initialize(message, cause)
+        @cause = cause
+        super(message)
+      end
+    end
+    class AuthenticationError < Error; end
+    class ConnectionError < Error; end
+    # Initialize TriglavClient
+    def initialize(opts = {})
+      @opts = opts || {}
+      config = TriglavClient::Configuration.new do |config|
+        uri = URI.parse(triglav_url)
+        config.scheme = uri.scheme
+        config.host = "#{uri.host}:#{uri.port}"
+        config.timeout = timeout if timeout
+        config.debugging = debugging if debugging
+      end
+      @api_client = TriglavClient::ApiClient.new(config)
+      initialize_current_token
+      authenticate
+    end
+    def close
+      # typhoeus makes a new connection and disconnects it on each request
+    end
+    # List resources required to be monitored
+    #
+    # @param [String] uri_prefix
+    # @return [Array of TriglavClient::ResourceEachResponse] array of resources
+    # @see TriglavClient::ResourceEachResponse
+    def list_aggregated_resources(uri_prefix)
+      $logger.debug { "ApiClient#list_aggregated_resources(#{uri_prefix.inspect})" }
+      resources_api = TriglavClient::ResourcesApi.new(@api_client)
+      handle_error { resources_api.list_aggregated_resources(uri_prefix) }
+    end
+    # Send messages
+    #
+    # @param [Array of TriglavClient::MessageRequest] array of event messages
+    # @see TriglavClient::MessageRequest
+    def send_messages(events)
+      $logger.debug { "ApiClient#send_messages(#{events.inspect})" }
+      messages_api = TriglavClient::MessagesApi.new(@api_client)
+      handle_error { messages_api.send_messages(events) }
+    end
+    def authorized?
+      @current_token.has_key?(:access_token)
+    end
+    private
+    # Authenticate
+    #
+    # 1. Another process saved a newer token onto the token_file => read it
+    # 2. The token saved on the token_file is same with current token => re-authenticate
+    # 3. The token saved on the token_file is older than the current token
+    #   => unknown situation, re-authenticate and save into token_file to refresh anyway
+    # 4. No token is saved on the token_file => authenticate
+    def authenticate
+      $logger.debug { 'ApiClient#authenticate' }
+      StorageFile.open(token_file) do |fp|
+        token = fp.load
+        if should_read_token_from_file?(token)
+          $logger.debug { "Read token from file" }
+          update_current_token(token)
+          return
+        end
+        $logger.debug { "AuthApi#create_token" }
+        auth_api = TriglavClient::AuthApi.new(@api_client)
+        credential = TriglavClient::Credential.new(
+          username: username, password: password, authenticator: authenticator
+        )
+        handle_auth_error do
+          $logger.debug { 'TriglavClient::AuthApi' }
+          result = auth_api.create_token(credential)
+          token = {access_token: result.access_token}
+          update_current_token(token)
+          fp.dump(token)
+        end
+      end
+    end
+    def initialize_current_token
+      @current_token = {
+        access_token: (@api_client.config.api_key['Authorization'] = String.new),
+      }
+    end
+    def update_current_token(token)
+      @current_token[:access_token].replace(token[:access_token])
+    end
+    def should_read_token_from_file?(token)
+      return true if @current_token[:access_token].empty? && !(token[:access_token] ||= '').empty?
+      return @current_token[:access_token] != token[:access_token]
+    end
+    def handle_auth_error(&block)
+      retries = 0
+      begin
+        yield
+      rescue TriglavClient::ApiError => e
+        if e.code == 0
+          if retries < max_retries
+            sleep retry_interval
+            retries += 1
+            retry
+          end
+          raise ConnectionError.new("Could not connect to #{triglav_url} with #{retries} retries", e)
+        elsif e.message == 'Unauthorized'.freeze
+          raise AuthenticationError.new("Failed to authenticate on triglav API.".freeze, e)
+        else
+          raise Error.new(e.message, e)
+        end
+      end
+    end
+    def handle_error(&block)
+      retries = 0
+      begin
+        yield
+      rescue TriglavClient::ApiError => e
+        if e.code == 0
+          if retries < max_retries
+            sleep retry_interval
+            retries += 1
+            retry
+          end
+          raise ConnectionError.new("Could not connect to #{triglav_url} with #{retries} retries", e)
+        elsif e.message == 'Unauthorized'.freeze
+          authenticate
+          retry
+        else
+          raise Error.new(e.message, e)
+        end
+      end
+    end
+    def triglav_url
+      @opts[:url] || $setting.dig(:triglav, :url)
+    end
+    def username
+      @opts.dig(:credential, :username) || $setting.dig(:triglav, :credential, :username)
+    end
+    def password
+      @opts.dig(:credential, :password) || $setting.dig(:triglav, :credential, :password)
+    end
+    def authenticator
+      @opts.dig(:credential, :authenticator) || $setting.dig(:triglav, :credential, :authenticator)
+    end
+    def timeout
+      @opts[:timeout] || $setting.dig(:triglav, :timeout)
+    end
+    def debugging
+      @opts[:debugging] || $setting.dig(:triglav, :debugging)
+    end
+    def max_retries
+      @opts[:retries] || $setting.dig(:triglav, :retries) || 3
+    end
+    def retry_interval
+      @opts[:retry_interval] || $setting.dig(:triglav, :retry_interval) || 3 # second
+    end
+    def token_file
+      @opts[:token_file] || $setting.token_file
+    end
+  end
+end

data/lib/triglav/agent/base/cli.rb ADDED Viewed

@@ -0,0 +1,98 @@
+require "triglav/agent/configuration"
+require "triglav/agent/base/setting"
+require 'optparse'
+require 'serverengine'
+module Triglav::Agent
+  module Base
+    # A base class for cli option parser
+    #
+    # This base class usually should be enough for agent plugins, but
+    # you can override this class and configure with Configuration#cli_classe=
+    class CLI
+      def run
+        opts, _ = parse_options(ARGV)
+        $setting = Configuration.setting_class.new(opts)
+        $logger = $setting.logger
+        se = ServerEngine.create(nil, Configuration.worker_module) do
+          $setting.serverengine_options
+        end
+        se.run
+      end
+      def default_opts
+        {
+          config: 'config.yml',
+          dotenv: false,
+          debug: false,
+        }
+      end
+      def option_parser(opts = {})
+        op = OptionParser.new
+        self.class.module_eval do
+          define_method(:usage) do |msg = nil|
+            puts op.to_s
+            puts "error: #{msg}" if msg
+            exit 1
+          end
+        end
+        op.on('-c', '--config VALUE', "Config file (default: #{opts[:config]})") {|v|
+          opts[:config] = v
+        }
+        op.on('-s', '--status VALUE', "Status stroage file (default: status.yml)") {|v|
+          opts[:status] = v
+        }
+        op.on('-t', '--token VALUE', "Triglav access token storage file (default: token.yml)") {|v|
+          opts[:token] = v
+        }
+        op.on('--dotenv', "Load environment variables from .env file (default: #{opts[:dotenv]})") {|v|
+          opts[:dotenv] = v
+        }
+        op.on('--debug', "Debug mode (default: #{opts[:debug]})") {|v|
+          opts[:debug] = v
+        }
+        op.on('-h', '--help', "help") {|v|
+          opts[:help] = v
+        }
+        # serverengine options
+        op.on('--log VALUE', "Log path (default: #{Setting::DEFAULT_LOG})") {|v|
+          opts[:log] = v
+        }
+        op.on('--log-level VALUE', "Log level (default: #{Setting::DEFAULT_LOG_LEVEL})") {|v|
+          opts[:log_level] = v
+        }
+        op.banner += ''
+        op
+      end
+      def parse_options(argv = ARGV)
+        opts = default_opts
+        op = option_parser(opts)
+        begin
+          args = op.parse(argv)
+        rescue OptionParser::InvalidOption => e
+          usage e.message
+        end
+        if opts[:help]
+          usage
+        end
+        if opts[:config].nil?
+          usage "--config VALUE is required"
+        end
+        if !File.readable?(opts[:config])
+          usage "Config file '#{opts[:config]}' does not exist or not readable"
+        end
+        [opts, args]
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/connection.rb ADDED Viewed

@@ -0,0 +1,30 @@
+module Triglav::Agent
+  module Base
+    # An abstract base class of Connection.
+    #
+    # Wrap a connection to a storage.
+    # You can implement any methods which you want to use in your Monitor class.
+    #
+    # You have to implement following methods:
+    #
+    # * initialize
+    # * close
+    #
+    # An instance is created for each `resource_uri_prefix`, that is,
+    # shared among resources with of same `resource_uri_prefix`.
+    #
+    # Note that multiple connection instances would be created
+    # when multiple parallel threads are created in Processor class.
+    class Connection
+      # @param [Hash] connection_info
+      def initialize(connection_info)
+        raise NotImplementedError
+      end
+      # close the connection
+      def close
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/monitor.rb ADDED Viewed

@@ -0,0 +1,32 @@
+module Triglav::Agent
+  module Base
+    # An abstract class of Monitor.
+    #
+    # Monitor your storage and send messages to triglav.
+    #
+    # You have to implement following methods:
+    #
+    # * initialize
+    # * process
+    #
+    # An instance is created per a `resource`.
+    # Connection is shared among same `resource_uri_prefix`.
+    #
+    # Note that multiple instances would be created,
+    # one instance for one parallel thread basically, and
+    # `#process` is ran concurrently.
+    class Monitor
+      # @param [Triglav::Agent::Base::Connection] connection
+      # @param [TriglavClient::ResourceResponse] resource
+      def initialize(connection, resource)
+        raise NotImplementedError
+      end
+      # @yield [events] Gives an array of events
+      def process(&block)
+        raise NotImplementedError
+        # yield(events) if block_given?
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/processor.rb ADDED Viewed

@@ -0,0 +1,136 @@
+require 'parallel'
+require 'connection_pool'
+module Triglav::Agent
+  module Base
+    # Triglav agent processor class.
+    #
+    # An instance is created for a `resource_uri_prefix`.
+    #
+    # You usually do not need to customize this class, but if you want to
+    # implement your original, configure
+    #
+    #     Triglav::Agent::Configuration.processor_class
+    class Processor
+      attr_reader :worker, :resource_uri_prefix
+      def initialize(worker, resource_uri_prefix)
+        @worker = worker
+        @resource_uri_prefix = resource_uri_prefix
+      end
+      def self.max_consecuitive_error_count
+        3
+      end
+      def process
+        before_process
+        success_count = 0
+        consecutive_error_count = 0
+        Parallel.each(resources, parallel_opts) do |resource|
+          raise Parallel::Break if stopped?
+          events = nil
+          begin
+            @connection_pool.with do |connection|
+              monitor = monitor_class.new(connection, resource)
+              monitor.process do |_events|
+                events = _events
+                $logger.info { "send_messages:#{events.map(&:to_hash).to_json}" }
+                @api_client_pool.with {|api_client| api_client.send_messages(events) }
+              end
+            end
+            @mutex.synchronize do
+              success_count += 1
+              consecutive_error_count = 0
+            end
+          rescue => e
+            log_error(e)
+            $logger.info { "failed_events:#{events.map(&:to_hash).to_json}" } if events
+            @mutex.synchronize do
+              raise TooManyError if (consecutive_error_count += 1) > self.class.max_consecuitive_error_count
+            end
+          end
+        end
+        success_count
+      ensure
+        after_process
+      end
+      def total_count
+        resources.size
+      end
+      private
+      def before_process
+        @connection_pool = ConnectionPool.new(connection_pool_opts) {
+          connection_class.new(get_connection_info(resource_uri_prefix))
+        }
+        @api_client_pool = ConnectionPool.new(connection_pool_opts) {
+          ApiClient.new # renew connection
+        }
+        @mutex = Mutex.new
+      end
+      def after_process
+        @connection_pool.shutdown {|conn| conn.close rescue nil }
+        @api_client_pool.shutdown {|conn| conn.close rescue nil }
+      end
+      def connection_class
+        Configuration.connection_class
+      end
+      def monitor_class
+        Configuration.monitor_class
+      end
+      def name
+        Configuration.name
+      end
+      def resources
+        return @resources if @resources
+        @resources = ApiClient.new.list_aggregated_resources(resource_uri_prefix) || []
+        $logger.debug { "resource_uri_prefix:#{resource_uri_prefix} resources.size:#{@resources.size}" }
+        @resources
+      end
+      def parallel_opts
+        parallel_type == 'process' ? {in_processes: parallel_size} : {in_threads: parallel_size}
+      end
+      def connection_pool_opts
+        {size: connection_pool_size, timeout: connection_pool_timeout}
+      end
+      def parallel_size
+        $setting.dig(name, :parallel_size) || 1
+      end
+      def parallel_type
+        $setting.dig(name, :parallel_type) || 'thread'
+      end
+      def connection_pool_size
+        $setting.dig(name, :connection_pool_size) || parallel_size
+      end
+      def connection_pool_timeout
+        $setting.dig(name, :connection_pool_timeout) || 60
+      end
+      def get_connection_info(resource_uri_prefix)
+        $setting.dig(name, :connection_info)[resource_uri_prefix]
+      end
+      def log_error(e)
+        $logger.error { "#{e.class} #{e.message} #{e.backtrace.join("\\n")}" } # one line
+      end
+      def stopped?
+        worker.stopped? if worker and worker.respond_to?(:stopped?)
+      end
+    end
+  end
+end