metrics-capacitor-engine 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2f466bef259df899e75151d5904a203924869aec
+  data.tar.gz: 986d0e7f768fa95088884dda0da44942eb2febbe
+SHA512:
+  metadata.gz: addf8bbb991ba5dcad978876d70f9ae69a5caa2f50a6d257e7cd5d7ba0b5e966a56a6f4522183bcb2bf80a6d8e98af1e51113255a0db4dadc108b46039b3b16a
+  data.tar.gz: 789b12ca634e3db7f65a34412f517d2b6b283c4c963ff7a5d9e9fd3aad41c054c63a04fc567472e6c9a2c63769341e2a093f50d30b47da77ab2c2ab7296bac1e

data/lib/metrics-capacitor.rb

@@ -0,0 +1,2 @@
+module MetricsCapacitor
+end

data/lib/metrics-capacitor/engine.rb

@@ -0,0 +1,138 @@
+require 'redis'
+require 'syslog'
+require 'syslog/logger'
+require 'metrics-capacitor/config'
+
+require_relative 'processor/core'
+require_relative 'processor/writer'
+require_relative 'processor/aggregator'
+require_relative 'processor/listener'
+
+module MetricsCapacitor
+  class Engine
+
+    def initialize
+      $0 = 'metrics-capacitor (engine)'
+      Config.load!
+      @exit_flag = false
+      @pids = []
+      @pids_kiq = []
+      %w(TERM INT).each do |sig|
+        Signal.trap(sig) do
+          @pids_kiq.each { |pid| Process.kill('INT', pid) rescue true }
+          @pids.each { |pid| Process.kill(sig, pid) rescue true }
+          Process.waitall
+          terminate_loggers
+        end
+      end
+      init_logger
+      log :info, "Engine warmed-up :-)"
+    end
+
+    def fork_processor(args = {})
+      log :debug, "Spawning #{args[:name]}"
+      args[:proc_num] ||= 1
+      args[:exit_on] ||= %w{INT TERM}
+      args[:proc_num].times do |num|
+        @logpipe["#{args[:name]}_#{num}".to_sym], logpipe = IO.pipe
+        @pids << Process.fork do
+          $0 = "metrics-capacitor (#{args[:name]})" if args[:name]
+          @logpipe["#{args[:name]}_#{num}".to_sym].close
+          remove_instance_variable(:@logpipe)
+          p = Kernel.const_get("MetricsCapacitor::Processor::#{args[:name].capitalize}").new(logpipe)
+          args[:exit_on].each { |sig| Signal.trap(sig) { p.shutdown! } }
+          p.start!
+        end
+        log :debug, "#{args[:name].capitalize} spawned as PID #{@pids.last.to_s}"
+        logpipe.close
+      end
+    end
+
+    def fork_scrubber
+      log :debug, "Spawning scrubbers"
+      Config.scrubber[:processes].times do |num|
+        @logpipe["scrubber_#{num}".to_sym], logpipe = IO.pipe
+        @pids_kiq << Process.fork do
+          @logpipe["scrubber_#{num}".to_sym].close
+          remove_instance_variable(:@logpipe)
+          require_relative 'sidekiq'
+          Sidekiq.configure_server do |config|
+            Sidekiq::Logging.logger = ::Logger.new(logpipe)
+            Sidekiq::Logging.logger.level = log_level
+            Sidekiq::Logging.logger.progname = "scrubber"
+            Sidekiq::Logging.logger.formatter = proc { |severity, _, progname, msg| "#{progname}##{Process.pid}|||#{severity}|||#{msg}\n" }
+            config.redis = { url: Config.redis[:url] }
+          end
+          Sidekiq.configure_client do |config|
+            config.redis = { url: Config.redis[:url] }
+          end
+          $TESTING = 0
+          kiq = Sidekiq::CLI.instance
+          kiq.parse(['-c', Config.scrubber[:threads].to_s, '-r', File.expand_path('..', __FILE__)+'/processor/scrubber.rb', '-q', 'scrubber'])
+          kiq.run
+        end
+        logpipe.close
+      end
+    end
+
+    def log(severity = :info, msg)
+      s = Kernel.const_get("Logger::#{severity.to_s.upcase}")
+      @logger_semaphore.synchronize do
+        @logger.log s, msg, 'engine#' + Process.pid.to_s
+      end
+    end
+
+    def log_level
+      Config.debug ? ::Logger::DEBUG : ::Logger::INFO
+    end
+
+    def init_logger
+      @logpipe = {}
+      @logger = ::Logger.new(STDOUT)
+      @logger.level = log_level
+      @logger.formatter = proc { |severity, datetime, progname, msg| [datetime.to_s, progname, severity, "#{msg}\n"].join(" ") }
+      @logger_threads = []
+      @logger_semaphore = Mutex.new
+    end
+
+    def spawn_logger_threads
+      @logpipe.each do |name, pipe|
+        @logger_threads << Thread.new do
+          Thread.current[:name] = "logger-#{name}"
+          while ! pipe.eof?
+            msg = pipe.gets
+            (progname,severity,message) = msg.split('|||')
+            @logger_semaphore.synchronize do
+              @logger.log Kernel.const_get("Logger::#{severity}"), message.chomp, progname
+            end
+          end
+        end
+      end
+    end
+
+    def terminate_loggers
+      @logger_threads.each(&:join)
+    end
+
+    def run!
+      log :info, 'Engine is starting up'
+      fork_scrubber
+      fork_processor name: 'writer', proc_num: Config.writer[:processes]
+      fork_processor name: 'aggregator'
+      fork_processor name: 'listener'
+      spawn_logger_threads
+      log :info, 'Engine has started :-)'
+      # TODO: unix socket for control and status reporting ;)
+      begin
+        ::Process.waitall
+        log :warn, 'Terminating!'
+      rescue Interrupt
+        retry
+      end
+      log :info, 'Terminating loggers'
+      terminate_loggers
+      log :warn, 'Engine is shutting down!'
+    end
+
+  end
+end

data/lib/metrics-capacitor/processor/aggregator.rb

@@ -0,0 +1,19 @@
+module MetricsCapacitor
+  module Processor
+    class Aggregator < Core
+
+      # def post_init
+      #   # TODO
+      # end
+      #
+      # def process
+      #   # TODO
+      # end
+      #
+      # def shutdown
+      #   # TODO
+      # end
+
+    end
+  end
+end

data/lib/metrics-capacitor/processor/core.rb

@@ -0,0 +1,63 @@
+require 'metrics-capacitor/model'
+
+module MetricsCapacitor
+  module Processor
+    class Core
+      include MetricsCapacitor::Model
+
+      def initialize(logpipe)
+        Config.load!
+        @_logger = ::Logger.new(logpipe)
+        @_name = self.class.to_s.split('::').last.downcase
+        @_logger.progname = @_name
+        @_logger.level = log_level
+        @_logger.formatter = proc { |severity, datetime, progname, msg| "#{progname}##{Process.pid}|||#{severity}|||#{msg}\n" }
+        logger.info "Initializing processor"
+        post_init
+      end
+
+      # implement in the processor Class
+      def post_init
+      end
+
+      # implement in the processor Class
+      def process
+        loop { logger.debug "alive"; sleep 10 }
+      end
+
+      # implement in the processor Class
+      def shutdown
+        exit 1
+      end
+
+      def logger
+        @_logger
+      end
+
+      def start!
+        logger.info "Starting processor"
+        begin
+          process
+        rescue StandardError => e
+          logger.fatal 'SHUTTING DOWN DUE TO AN EXCEPTION!'
+          logger.fatal [e.class, e.message].join(' -> ')
+          logger.fatal e.backtrace
+          logger.fatal 'Sending SIGINT to the engine'
+          Process.kill('INT', Process.ppid)
+          shutdown!
+        end
+        logger.info "Processor finished"
+      end
+
+      def shutdown!
+        shutdown
+      end
+
+      private
+      def log_level
+        Config.debug ? ::Logger::DEBUG : ::Logger::INFO
+      end
+
+    end
+  end
+end

data/lib/metrics-capacitor/processor/listener.rb

@@ -0,0 +1,19 @@
+module MetricsCapacitor
+  module Processor
+    class Listener < Core
+
+      # def post_init
+      #   # TODO
+      # end
+      #
+      # def process
+      #   # TODO
+      # end
+      #
+      # def shutdown
+      #   # TODO
+      # end
+
+    end
+  end
+end

data/lib/metrics-capacitor/processor/scrubber.rb

@@ -0,0 +1,30 @@
+require 'metrics-capacitor/config'
+require 'metrics-capacitor/model'
+
+module MetricsCapacitor
+  module Processor
+    class Scrubber
+      include Sidekiq::Worker
+      include MetricsCapacitor::Model
+
+      sidekiq_options retry: true
+      sidekiq_options queue: :scrubber
+      sidekiq_options backtrace: true
+
+      REDIS = ConnectionPool.new(size: 2) { Redis.new(url: Config.redis[:url]) }
+
+      def perform *args
+        logger.debug 'Picking redis client from connection-pool'
+        REDIS.with do |r|
+          logger.debug 'Parsing metrics data'
+          metrics = Metrics.new args
+          logger.debug metrics.to_redis
+          logger.debug 'Sending data to writer'
+          r.rpush 'writer', metrics.to_redis
+          logger.debug 'Data sent'
+        end
+      end
+
+    end
+  end
+end

data/lib/metrics-capacitor/processor/writer.rb

@@ -0,0 +1,67 @@
+require 'elasticsearch'
+
+module MetricsCapacitor
+  module Processor
+    class Writer < Core
+
+      def post_init
+        @elastic = Elasticsearch::Client.new(
+          url: Config.elasticsearch[:urls],
+          reload_connections: 100,
+          retry_on_failure: Config.elasticsearch[:retry],
+          sniffer_timeout: 5,
+        )
+        logger.debug 'Elastic connection set up'
+
+        @redis = Redis.new(url: Config.redis[:url])
+        logger.debug 'Redis connection set up'
+
+        logger.debug 'Setting ES index templates'
+        @elastic.indices.put_template name: 'metrics', body: INDEX_TEMPLATE
+        logger.debug 'ES templates set up'
+
+        @exit = false
+      end
+
+      def process
+        logger.debug 'Randomizing startup time'
+        sleep rand(Config.writer[:bulk_wait])
+        until @exit
+          logger.debug 'Gathering mertics bulk'
+          metric = nil
+          metrics = Metrics.new
+          indexing_result = nil
+          begin
+            while !@exit && metrics.length < Config.writer[:bulk_max] && ( metric = @redis.blpop('writer', timeout: Config.writer[:bulk_wait]) )
+              metrics << Metric.new(metric[1])
+              metric = nil
+            end
+          rescue Redis::CannotConnectError, Redis::TimeoutError => e
+            logger.error "Can't connect to redis: #{e.message}"
+            sleep 1
+            retry
+          end
+          unless metrics.empty?
+            logger.debug "Writing #{metrics.length} metrics"
+            logger.debug metrics.to_elastic.to_json
+            indexing_result = @elastic.bulk(index: Time.now.strftime(Config.elasticsearch[:index]), type: Config.writer[:doc_type], body: metrics.to_elastic)
+            if indexing_result['errors']
+              logger.error 'Failed to write metrics!'
+              logger.error indexing_result['items'].to_json
+            else
+              logger.info "Written #{metrics.length} metrics, took #{indexing_result['took']}ms"
+            end
+          else
+            logger.warn 'No metrics to write :-('
+          end
+          metrics = nil
+          indexing_result = nil
+        end
+      end
+
+      def shutdown
+        @exit = true
+      end
+    end
+  end
+end

data/lib/metrics-capacitor/sidekiq.rb

@@ -0,0 +1,49 @@
+require 'sidekiq'
+require 'sidekiq/cli'
+require 'sidekiq/logging'
+
+module Sidekiq
+  class CLI
+
+    PROCTITLES[0] = proc { 'metrics-capacitor'.freeze }
+    PROCTITLES[1] = proc { '(scrubber)'.freeze }
+
+    def run
+      @code = nil
+
+      boot_system
+
+      self_read, self_write = IO.pipe
+
+      %w(INT USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+
+      ver = Sidekiq.redis_info['redis_version']
+      raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+      fire_event(:startup)
+      logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
+      logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
+      require 'sidekiq/launcher'
+      @launcher = Sidekiq::Launcher.new(options)
+      begin
+        launcher.run
+        while readable_io = IO.select([self_read])
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        logger.info 'Shutting down'
+        launcher.stop
+        logger.info "Bye!"
+        exit(0)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: metrics-capacitor-engine
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Radek 'blufor' Slavicinsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: metrics-capacitor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.1.2
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.17
+description: Engine (workers) for Metrics Capacitor
+email: radek@blufor.cz
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/metrics-capacitor.rb
+- lib/metrics-capacitor/engine.rb
+- lib/metrics-capacitor/processor/aggregator.rb
+- lib/metrics-capacitor/processor/core.rb
+- lib/metrics-capacitor/processor/listener.rb
+- lib/metrics-capacitor/processor/scrubber.rb
+- lib/metrics-capacitor/processor/writer.rb
+- lib/metrics-capacitor/sidekiq.rb
+homepage: https://github.com/metrics-capacitor/metrics-capacitor-engine
+licenses:
+- GPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Metrics Capacitor (engine)
+test_files: []