triglav-agent 1.0.0.pre1

data/lib/triglav/agent/base/setting.rb

@@ -0,0 +1,112 @@
+require 'triglav/agent/hash_util'
+require 'triglav/agent/logger'
+require 'yaml'
+require 'erb'
+
+module Triglav::Agent
+  module Base
+    # A base class represents settings coming from config.yml and cli options
+    #
+    # This base class usually should be enough for agent plugins, but
+    # you can override this class and configure with Configuration#setting_class=
+    class Setting
+      attr_reader :cli_options
+
+      DEFAULT_LOG = 'STDOUT'.freeze
+      DEFAULT_LOG_LEVEL = 'info'.freeze
+      DEFAULT_APP_ENV = 'development'.freeze
+
+      def initialize(cli_options = {})
+        @cli_options = cli_options
+        if dotenv?
+          require 'dotenv'
+          Dotenv.load
+        end
+      end
+
+      def reload
+        Dotenv.overload if dotenv?
+        @config = nil
+        @logger.close rescue nil
+        @logger = nil
+      end
+
+      def app_env
+        @app_env ||= ENV['APP_ENV'] || DEFAULT_APP_ENV
+      end
+
+      def config_file
+        @cli_options[:config]
+      end
+
+      def status_file
+        @status_file ||= @cli_options[:status] || config.dig(:triglav, :status_file) || 'status.yml'
+      end
+
+      def token_file
+        @token_file ||= @cli_options[:token] || config.dig(:triglav, :token_file) || 'token.yml'
+      end
+
+      def dotenv?
+        @cli_options[:dotenv]
+      end
+
+      def debug?
+        @cli_options[:debug]
+      end
+
+      def logger
+        @logger ||= Logger.new(log, serverengine_logger_options)
+      end
+
+      def log_level
+        @cli_options[:log_level] || config.dig(:serverengine, :log_level) || DEFAULT_LOG_LEVEL
+      end
+
+      def log
+        @cli_options[:log] || config.dig(:serverengine, :log) || DEFAULT_LOG
+      end
+
+      def serverengine_options
+        serverengine_options = config[:serverengine].dup || {}
+        # serverengine_options[:workers] = 1 # default
+        # serverengine_options[:worker_type] = 'embedded' # default
+        serverengine_options.keys.each do |k|
+          serverengine_options.delete(k) if k.to_s.start_with?('log')
+        end
+        serverengine_options.merge!({
+          logger: logger,
+          log_level: log_level,
+          setting: self,
+        })
+      end
+
+      def [](key)
+        config[key]
+      end
+
+      def dig(*args)
+        config.dig(*args)
+      end
+
+      private
+
+      def config
+        @config ||=
+          begin
+            raw = File.read(config_file)
+            erb = ERB.new(raw, nil, "-").tap {|_| _.filename = config_file }
+            all = HashUtil.deep_symbolize_keys(YAML.load(erb.result(binding)))
+            all[app_env.to_sym]
+          end
+      end
+
+      def serverengine_logger_options
+        {
+          log: log,
+          log_level: log_level,
+        }
+      end
+    end
+  end
+end

data/lib/triglav/agent/base/worker.rb

@@ -0,0 +1,95 @@
+require 'triglav/agent/timer'
+
+module Triglav::Agent
+  module Base
+    # Triglav agent worker module for Serverengine.
+    #
+    # You usually do not need to customize this module, but if you want to
+    # implement your original, configure
+    #
+    #     Triglav::Agent::Configuration.worker_module
+    module Worker
+      # serverengine interface
+      def initialize
+        @timer = Timer.new
+      end
+
+      # serverengine interface
+      def reload
+        $logger.info { "Worker#reload worker_id:#{worker_id}" }
+        $setting.reload
+      end
+
+      # serverengine interface
+      def run
+        $logger.info { "Worker#run worker_id:#{worker_id}" }
+        start
+        until @stop
+          @timer.wait(monitor_interval) { process }
+        end
+      rescue => e
+        # ServerEngine.dump_uncaught_error does not tell me e.class
+        log_error(e)
+        raise e
+      end
+
+      def process
+        started = Time.now
+        $logger.info { "Start Worker#process worker_id:#{worker_id}" }
+
+        total_count = 0
+        total_success_count = 0
+        resource_uri_prefixes.each do |resource_uri_prefix|
+          break if stopped?
+          processor = processor_class.new(self, resource_uri_prefix)
+          total_count += processor.total_count
+          total_success_count += processor.process
+        end
+
+        elapsed = Time.now - started
+        $logger.info {
+          "Finish Worker#process worker_id:#{worker_id} " \
+          "success_count/total_count:#{total_success_count}/#{total_count} elapsed:#{elapsed.to_f}sec"
+        }
+      end
+
+      def start
+        @timer.start
+        @stop = false
+      end
+
+      # serverengine interface
+      def stop
+        $logger.info { "Worker#stop worker_id:#{worker_id}" }
+        @stop = true
+        @timer.stop
+      end
+
+      def stopped?
+        @stop
+      end
+
+      private
+
+      def processor_class
+        Configuration.processor_class
+      end
+
+      def name
+        Configuration.name
+      end
+
+      def log_error(e)
+        $logger.error { "#{e.class} #{e.message} #{e.backtrace.join("\\n")}" } # one line
+      end
+
+      def monitor_interval
+        $setting.dig(name, :monitor_interval) || 60
+      end
+
+      def resource_uri_prefixes
+        $setting.dig(name, :connection_info).keys
+      end
+    end
+  end
+end

data/lib/triglav/agent/configuration.rb

@@ -0,0 +1,79 @@
+module Triglav
+  module Agent
+    # Configure Triglav::Agent framework
+    #
+    #    require 'triglav/agent/configuration'
+    #    require 'triglav/agent/vertica/worker'
+    #    Triglav::Agent::Configuration.configure do |config|
+    #      config.name = :vertica # default: :agent
+    #      config.cli_class = Triglav::Agent::Vertica::CLI
+    #      config.setting_class = Triglav::Agent::Vertica::Setting
+    #      config.worker_module = Triglav::Agent::Vertica::Worker
+    #      config.processor_class = Triglav::Agent::Vertica::Processor
+    #      config.monitor_class = Triglav::Agent::Vertica::Monitor
+    #      config.connection_class = Triglav::Agent::Vertica::Connection
+    #    end
+    #    Triglav::Agent::Configuration.cli_class.new.run
+    class Configuration
+      def self.configure(&block)
+        yield(Triglav::Agent::Configuration)
+      end
+
+      def self.worker_module
+        @worker_module ||= Triglav::Agent::Base::Worker
+      end
+
+      def self.processor_class
+        @processor_class ||= Triglav::Agent::Base::Processor
+      end
+
+      def self.monitor_class
+        @monitor_class ||= Triglav::Agent::Base::Monitor
+      end
+
+      def self.connection_class
+        @connection_class ||= Triglav::Agent::Base::Connection
+      end
+
+      def self.setting_class
+        @setting_class ||= Triglav::Agent::Base::Setting
+      end
+
+      def self.cli_class
+        @cli_class ||= Triglav::Agent::Base::CLI
+      end
+
+      def self.name
+        @name ||= :agent
+      end
+
+      def self.worker_module=(worker_module)
+        @worker_module = worker_module
+      end
+
+      def self.processor_class=(processor_class)
+        @processor_class = processor_class
+      end
+
+      def self.monitor_class=(monitor_class)
+        @monitor_class = monitor_class
+      end
+
+      def self.connection_class=(connection_class)
+        @connection_class = connection_class
+      end
+
+      def self.setting_class=(setting_class)
+        @setting_class = setting_class
+      end
+
+      def self.cli_class=(cli_class)
+        @cli_class = cli_class
+      end
+
+      def self.name=(name)
+        @name = name
+      end
+    end
+  end
+end

data/lib/triglav/agent/error.rb

@@ -0,0 +1,4 @@
+module Triglav::Agent
+  class Error < ::StandardError; end
+  class TooManyError < Error; end
+end

data/lib/triglav/agent/hash_util.rb

@@ -0,0 +1,36 @@
+module Triglav::Agent
+  class HashUtil
+    def self.deep_symbolize_keys(obj)
+      case obj
+      when Hash
+        obj.map {|k, v| [k.to_sym, deep_symbolize_keys(v)] }.to_h
+      when Array
+        obj.map {|v| deep_symbolize_keys(v) }
+      else
+        obj
+      end
+    end
+
+    def self.deep_stringify_keys(obj)
+      case obj
+      when Hash
+        obj.map {|k, v| [k.to_s, deep_stringify_keys(v)] }.to_h
+      when Array
+        obj.map {|v| deep_stringify_keys(v) }
+      else
+        obj
+      end
+    end
+
+    #   HashUtil.setdig(hash, ['a', 'b'], 'bar') # like hash['a']['b'] = 'bar'
+    def self.setdig(hash, key, val)
+      keys = Array(key)
+      sub_hash = hash
+      keys[0...-1].each do |k|
+        sub_hash = (sub_hash[k] ||= {})
+      end
+      sub_hash[keys.last] = val
+      hash
+    end
+  end
+end

data/lib/triglav/agent/logger.rb

@@ -0,0 +1,50 @@
+require 'serverengine/daemon_logger'
+
+module Triglav::Agent
+  # Logger class
+  #
+  #     require 'triglav/agent/logger'
+  #     logger = Logger.new('STDOUT', opts = {})
+  class Logger < ::ServerEngine::DaemonLogger
+    def initialize(logdev, *args)
+      logdev = STDOUT if logdev == 'STDOUT'
+      logdev = STDERR if logdev == 'STDERR'
+      super(logdev, *args)
+      @formatter = LogFormatter.new
+    end
+
+    def write(msg)
+      @logdev.write msg
+    end
+  end
+
+  class LogFormatter
+    FORMAT = "%s [%s] %s %s\n"
+
+    def initialize(opts={})
+    end
+
+    def call(severity, time, progname, msg)
+      FORMAT % [format_datetime(time), severity, format_pid, format_message(msg)]
+    end
+
+    private
+    def format_datetime(time)
+      time.strftime("%Y-%m-%dT%H:%M:%S.%6N%:z")
+    end
+
+    def format_pid
+      "PID-#{::Process.pid} TID-#{Thread.current.object_id}"
+    end
+
+    def format_message(message)
+      case message
+      when ::Exception
+        e = message
+        "#{e.class} (#{e.message})\\n  #{e.backtrace.join("\\n  ")}"
+      else
+        message.to_s.gsub(/\n/, "\\n")
+      end
+    end
+  end
+end

data/lib/triglav/agent/storage_file.rb

@@ -0,0 +1,144 @@
+require 'triglav/agent/hash_util'
+require 'yaml'
+
+module Triglav::Agent
+  # Thread and inter-process safe YAML file storage
+  #
+  #     StorageFile.open($setting.status_file) do |fp|
+  #       status = fp.load
+  #       status['foo'] = 'bar'
+  #       fp.dump(status)
+  #     end
+  class StorageFile
+    attr_reader :fp
+
+    private def initialize(fp)
+      @fp = fp
+    end
+
+    # Load storage file
+    #
+    #     StorageFile.load($setting.status_file)
+    #
+    # @param [String] path
+    # @return [Hash]
+    def self.load(path)
+      open(path) {|fp| fp.load }
+    end
+
+    # Open storage file
+    #
+    #     StorageFile.open($setting.status_file) do |fp|
+    #       status = fp.load
+    #       status['foo'] = 'bar'
+    #       fp.dump(status)
+    #     end
+    #
+    # @param [String] path
+    # @param [Block] block
+    def self.open(path, &block)
+      fp = File.open(path, (File::RDONLY | File::CREAT))
+      until fp.flock(File::LOCK_EX | File::LOCK_NB)
+        $logger.info { "Somebody else is locking the storage file #{path.inspect}" }
+        sleep 0.5
+      end
+      begin
+        return yield(StorageFile.new(fp))
+      ensure
+        fp.flock(File::LOCK_UN)
+        fp.close rescue nil
+      end
+    end
+
+    # Set storage file with given key, value
+    #
+    #     StorageFile.set($setting.status_file, 'foo', 'bar') # like h['foo'] = 'bar'
+    #     StorageFile.set($setting.status_file, ['a','b'], 'bar') # like h['a']['b'] = 'bar'
+    #
+    # @param [String] path
+    # @param [Object] key
+    # @param [Object] val
+    def self.set(path, key, val)
+      keys = Array(key)
+      open(path) do |fp|
+        params = fp.load
+        HashUtil.setdig(params, keys, val)
+        fp.dump(params)
+      end
+    end
+
+    # Set key to hold val if key does not exist
+    #
+    #     StorageFile.setnx($setting.status_file, 'foo', 'bar') # like h['foo'] = 'bar'
+    #     StorageFile.setnx($setting.status_file, ['a','b'], 'bar') # like h['a']['b'] = 'bar'
+    #
+    # @param [String] path
+    # @param [Object] key
+    # @param [Object] val
+    # @return [Boolean] true if set (not exist), false if not set (exists)
+    def self.setnx(path, key, val)
+      keys = Array(key)
+      open(path) do |fp|
+        params = fp.load
+        return false if params.dig(*keys)
+        HashUtil.setdig(params, keys, val)
+        fp.dump(params)
+        return true
+      end
+    end
+
+    # Set key to hold val if key does not exist and returns the holded value
+    #
+    # This is a kind of atomic short hand of
+    #
+    #     StorageFile.setnx($setting.status_file, 'foo', 'bar')
+    #     StorageFile.get($setting.status_file, 'foo')
+    #
+    # @param [String] path
+    # @param [Object] key
+    # @param [Object] val
+    # @return [Object] holded value
+    def self.getsetnx(path, key, val)
+      keys = Array(key)
+      open(path) do |fp|
+        params = fp.load
+        if curr = params.dig(*keys)
+          return curr
+        end
+        HashUtil.setdig(params, keys, val)
+        fp.dump(params)
+        return val
+      end
+    end
+
+    # Get value of the given key from storage file
+    #
+    #     StorageFile.get($setting.status_file, 'foo') # like h['foo'] = 'bar'
+    #     StorageFile.get($setting.status_file, ['a','b']) # like hash['a']['b']
+    #
+    # @param [String] path
+    # @param [Object] key
+    def self.get(path, key)
+      keys = Array(key)
+      open(path) {|fp| fp.load.dig(*keys) }
+    end
+
+    # Load storage file
+    #
+    # @return [Hash]
+    def load
+      if !(content = @fp.read).empty?
+        YAML.load(content) # all keys must be symbols
+      else
+        {}
+      end
+    end
+
+    # Dump to storage file
+    #
+    # @param [Hash] hash
+    def dump(hash)
+      File.write(@fp.path, YAML.dump(hash))
+    end
+  end
+end

data/lib/triglav/agent/timer.rb

@@ -0,0 +1,80 @@
+module Triglav::Agent
+  # A timer utility to run serverengine worker in a time interval
+  #
+  #     module Triglav::Agent
+  #       module Worker
+  #         def initialize
+  #           @interval = 60.0 # sec
+  #         end
+  #
+  #         def run
+  #           @timer = Timer.new
+  #           @stop = false
+  #           until @stop
+  #             @timer.wait(@interval) { process }
+  #           end
+  #         end
+  #
+  #         def stop
+  #           @stop = true
+  #           @timer.stop
+  #         end
+  #       end
+  #     end
+  class Timer
+    def initialize
+      @r, @w = IO.pipe
+      start
+    end
+
+    def wait(sec, &block)
+      return if @stop
+      started = Time.now
+      yield
+      elapsed = Time.now - started
+      if (timeout = (sec - elapsed).to_f) > 0
+        begin
+          IO.select([@r], [], [], timeout)
+        rescue IOError, Errno::EBADF
+          # these error may occur if @r is closed during IO.select. Ignore it
+        end
+      end
+    end
+
+    def start
+      @stop = false
+    end
+
+    def stop
+      @stop = true
+      signal
+    end
+
+    private
+
+    def signal
+      @w.puts ' '
+    end
+
+    # # Hmm, Ctrl-C breaks condvar.wait before calling #stop unexpectedly
+    # attr_reader :condvar, :mutex
+    #
+    # def initialize
+    #   @condvar = ConditionVariable.new
+    #   @mutex = Mutex.new
+    # end
+    #
+    # def wait(sec, &block)
+    #   started = Time.now
+    #   @mutex.synchronize do
+    #     yield
+    #     elapsed = (Time.now - started).to_f
+    #     @condvar.wait(@mutex, sec - elapsed)
+    #   end
+    # end
+    #
+    # def signal
+    #   @condvar.signal
+    # end
+  end
+end

data/lib/triglav/agent/version.rb

@@ -0,0 +1,5 @@
+module Triglav
+  module Agent
+    VERSION = "1.0.0.pre1"
+  end
+end

data/lib/triglav/agent.rb

@@ -0,0 +1,20 @@
+module Triglav
+  module Agent
+  end
+end
+
+require 'triglav/agent/api_client'
+require 'triglav/agent/configuration'
+require 'triglav/agent/error'
+require 'triglav/agent/hash_util'
+require 'triglav/agent/logger'
+require 'triglav/agent/storage_file'
+require 'triglav/agent/timer'
+require 'triglav/agent/version'
+
+require 'triglav/agent/base/cli'
+require 'triglav/agent/base/setting'
+require 'triglav/agent/base/worker'
+require 'triglav/agent/base/processor'
+require 'triglav/agent/base/connection' # just a skelton
+require 'triglav/agent/base/monitor' # just a skelton

data/lib/triglav-agent.rb

@@ -0,0 +1 @@
+require 'triglav/agent'

data/triglav-agent.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'triglav/agent/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "triglav-agent"
+  spec.version       = Triglav::Agent::VERSION
+  spec.authors       = ["Triglav Team"]
+  spec.email         = ["triglav_admin_my@dena.jp"]
+
+  spec.summary       = %q{Framework of Triglav Agent in Ruby.}
+  spec.description   = %q{Framework of Triglav Agent in Ruby.}
+  spec.homepage      = "https://github.com/triglav-dataflow/triglav-agent-framework-ruby"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "serverengine"
+  spec.add_dependency "dotenv"
+  spec.add_dependency "triglav_client"
+  spec.add_dependency "parallel"
+  spec.add_dependency "connection_pool"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
+  spec.add_development_dependency "yard"
+end