pushyd 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4c0863d93fc6a8efc1323791dcab4cde429ff67
-  data.tar.gz: ebbd792bf40400fbca51905c61611707b2013b3e
+  metadata.gz: da7515bbd91875e7d4d0c28edcd7b1222ddeb8a2
+  data.tar.gz: 633a9271a1359e036a00d94cf800ad458744f1da
 SHA512:
-  metadata.gz: abca867f07fe275efc2ffe00c26098760a8a4ce2d8e80b240f3be1a8e068df37e4b12eec29a0d61f51a587d9c5ce045f5441916805764a6cb4d96f25c1003270
-  data.tar.gz: da02db1407df734c58854bd123330c22da5c1ab3a200cbbf721485b4e85db3fa31ea67662f3898bff476bf2055588ff42acb4c4b08d5585155047fb4e7f27a2b
+  metadata.gz: faf3ee6045545e77dc262d0b1b73469ea4531bf690ec3536cd367b33a02c5a9c765b23dcc3e615636a5eca47225a485cd80b54ad052b35ec2ee6d35207225269
+  data.tar.gz: ad8edf30c5fa35fc447ad0e51347ad7479d24ca6533457d597701e6dbb46d8ef695f21413be049baad855314dcaeafb963a5d6a9cd97bcddda98989aba507c84

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pushyd (0.3.4)
+    pushyd (0.4.0)
       bunny
       chamber
       daemons

data/bin/pushyd

@@ -5,7 +5,6 @@ begin
   require "rubygems"
   require "optparse"
   require 'daemons'
-  require 'logger'
   require_relative "../lib/pushyd/conf"
 rescue LoadError
   raise "EXITING: some basic libs were not found"
@@ -33,14 +32,14 @@ begin
   end.order!(ARGV)
 
   # Build Chamber-based configuration from Gemspec with initial context
-  Conf.prepare root: APP_ROOT, gemspec: "pushyd", env: cmd_env, config: cmd_config, log: cmd_logfile
+  Conf.prepare root: APP_ROOT, gemspec: "pushyd", env: cmd_env, config: cmd_config, logfile: cmd_logfile
 
 rescue OptionParser::InvalidOption => e
-  abort "EXITING: option parser: #{e.message}"
+  abort "EXITING: InvalidOption: #{e.message} \n #{e.backtrace.to_yaml}"
 rescue PushyDaemon::ConfigParseError => e
-  abort "EXITING: ConfigParseError: #{e.message}"
+  abort "EXITING: ConfigParseError: #{e.message} \n #{e.backtrace.to_yaml}"
 rescue StandardError => e
-  abort "EXITING: Exception: #{e.message}"
+  abort "EXITING: StandardError: #{e.message} \n #{e.backtrace.to_yaml}"
 end
 
 # Display final configuration
@@ -48,8 +47,7 @@ puts "--- #{Conf.name} #{Conf.version}"
 puts "YAML Parser    \t #{YAML.name}"
 puts "Environment    \t #{Conf.env}"
 puts "Config files   \t #{Conf.files}"
-puts "Loging to file \t #{Conf[:log]}"
-
+puts "Loging to file \t #{Conf.log.logfile}"
 
 # Quit if config dump requested
 if cmd_dump
@@ -65,10 +63,10 @@ run_options = {
   }
 Daemons.run_proc('pushy-daemon', run_options) do
   # Load code
-  puts "--- loading code"
+  puts "--- load code"
   require_relative "../lib/pushyd"
 
   # Start daemon
-  puts "--- starting daemon"
+  puts "--- start daemon"
   PushyDaemon::Daemon.run
 end

data/defaults.yml

@@ -6,7 +6,9 @@ bus:
   user: guest
   pass: guest
 
-log: null
+log:
+  file: null
+  level: debug
 
 shout:
   topic: pushyd
@@ -19,6 +21,13 @@ shout:
   #   - updated
   #   - deleted
   #   - crunched
+
+# newrelic:
+#   licence: ""
+#   platform: "bigbusiness"               # app platform name
+#   app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
+#   logfile: null
+
 # rules:
 #   proxy_tests:
 #     title: All proxy tests messages
@@ -26,3 +35,4 @@ shout:
 #     routes: "proxy.#"
 #     subscribe: false
 #     #relay: http://requestb.in/1clzv7v1
+

data/lib/pushyd/conf.rb

@@ -2,6 +2,7 @@ require "chamber"
 
 module PushyDaemon
   class ConfigMissingParameter    < StandardError; end
+  class ConfigOtherError          < StandardError; end
   class ConfigParseError          < StandardError; end
 
   class Conf
@@ -13,6 +14,7 @@ module PushyDaemon
       attr_reader :files
       attr_reader :version
       attr_reader :env
+      attr_reader :host
     end
 
     def self.prepare args = {}
@@ -25,6 +27,9 @@ module PushyDaemon
       fail PushyDaemon::ConfigMissingParameter, "missing gemspec" unless args[:gemspec]
       fail PushyDaemon::ConfigMissingParameter, "gemspec file not found: #{gemspec_path}" unless File.exist?(gemspec_path)
 
+      # Init host if missing
+      @host ||= `hostname`.to_s.chomp.split(".").first
+
       # Load Gemspec
       @spec     = Gem::Specification::load gemspec_path
       @name     = @spec.name
@@ -39,23 +44,61 @@ module PushyDaemon
       # Load configuration files
       load files: @files, namespaces: { environment: @env }
 
+      # Override some values
+      self[:log] ||= {}
+      if args[:logfile]
+        self[:log][:file] = logfile
+      end
+
+      # Init New Relic
+      prepare_newrelic self[:newrelic]
+
       # Try to access any key to force parsing of the files
       self[:dummy]
 
-      # Override some values
-      self[:log] = args[:log].to_s if args[:log]
-
     rescue Psych::SyntaxError => e
       fail PushyDaemon::ConfigParseError, e.message
-
-    rescue Exception => e
-      fail PushyDaemon::ConfigParseError, e.message
-
+    rescue StandardError => e
+      fail PushyDaemon::ConfigOtherError, "#{e.message} \n #{e.backtrace.to_yaml}"
     end
 
     def self.dump
       self.to_hash.to_yaml
     end
 
+    def Conf.newrelic_enabled?
+      !!self[:newrelic]
+    end
+
+  protected
+
+    def self.prepare_newrelic section
+      unless section.is_a?(Hash)
+        puts "prepare_newrelic: no config"
+        ENV["NEWRELIC_AGENT_ENABLED"] = "false"
+        return
+      end
+        puts "prepare_newrelic: config ok"
+
+      # Enable GC profiler
+      GC::Profiler.enable
+
+      # Enable module
+      ENV["NEWRELIC_AGENT_ENABLED"] = "true"
+      ENV["NEW_RELIC_MONITOR_MODE"] = "true"
+
+      # License
+      ENV["NEW_RELIC_LICENSE_KEY"] = section[:licence].to_s
+
+      # Appname
+      platform = section[:platform] || self.host
+      section[:app_name] ||= "#{self.name}-#{platform}-#{self.env}"
+      ENV["NEW_RELIC_APP_NAME"] = section[:app_name].to_s
+
+      # Logfile
+      ENV["NEW_RELIC_LOG"] = section[:logfile].to_s
+    end
+
+
   end
 end

data/lib/pushyd/constants.rb

@@ -4,14 +4,14 @@ WAY_IN    = "RECD"
 WAY_POST  = "POST"
 
 # Constants: proxy
-PROXY_MESSAGE_MAX   = 1
-PROXY_USE_ACK       = false
+PROXY_MESSAGE_MAX       = 1
+PROXY_USE_ACK           = false
 
 # Constants: logger
-LOG_COL_ID              = 6
+LOG_ROTATION            = "daily"
 LOG_TRIM_LINE           = 200
 LOG_FORMAT_TIME         = "%Y-%m-%d %H:%M:%S"
-LOG_FORMAT_MESSAGE      = "%-6s"
-LOG_NEWLINE             = "\n"
-LOG_INDENT              = "\t"
-LOG_ROTATION            = "daily"
+LOG_FORMAT_LINE         = "%s\t%-8s\t%-10s\t%s\n"
+LOG_FORMAT_ARRAY        = "%s\t%-8s\t%-10s\t   %s\n"
+LOG_FORMAT_HASH         = "%s\t%-8s\t%-10s\t   %-17s %s\n"
+

data/lib/pushyd/daemon.rb

@@ -19,7 +19,7 @@ module PushyDaemon
       end
 
     rescue Errno::EACCES, StandardError => e
-      abort "EXITING #{e.class}: #{e.message}"
+      abort "EXITING #{e.class}: #{e.message} \n #{e.backtrace.to_yaml}"
     end
 
   end

data/lib/pushyd/endpoint.rb

@@ -10,33 +10,70 @@ module PushyDaemon
   class Endpoint
 
     def initialize
-      # Prepare logger (may be NIL > won't output anything)
-      logfile = Conf[:log]
+      # Prepare logger
+      init_logger Conf[:log]
+
+      # Done
+      info "endpoint initialized"
+      # loop do
+      #   info "info"
+      #   info ["info1", "info2", "info3"]
+      #   error "error"
+      #   debug "debug"
+      #   sleep 1
+      # end
+    end
+
+  protected
 
-      # Create the logger
-      @logger = PushyLogger.new(logfile, LOG_ROTATION)
-      @logger.add Logger::INFO, "starting #{self.class.name}"
+    def init_logger logconf
+      # Extract context
+      logfile   = logconf[:file]
+      loglevel  = logconf[:level]
+      me        = self.class.name
+
+      # Prepare logger (may be NIL > won't output anything)
+      @logger = Logger.new(logfile, LOG_ROTATION)
+      @logger.formatter = Formatter
+
+      # Set progname
+      @logger.progname = me.split('::').last
+
+      # Set expected level
+      @logger.level = case loglevel
+      when "debug"
+        Logger::DEBUG
+      when "info"
+        Logger::INFO
+      when "warn"
+        Logger::WARN
+      else
+        Logger::INFO
+      end
 
-      # Declare we're now logging
-      puts "#{self.class} logging to #{logfile}"
+      # Announce on STDOUT we're now logging to file
+      if logfile
+        puts "#{self.class} logging loglevel [#{loglevel} > #{@logger.level}] to [#{logfile}]"
+      else
+        puts "#{self.class} logging disabled"
+      end
     end
 
-  protected
+    def error messages
+      @logger.error messages
+    end
 
-    def error message, lines = {}
-      @logger.add Logger::ERROR, "#{self.class}: #{message}", lines
+    def info messages
+      @logger.info messages
     end
 
-    def info message, lines = {}
-      @logger.add Logger::INFO, "#{self.class}: #{message}", lines
+    def debug messages
+      @logger.debug messages
     end
 
     def message params = {}
-      # Indenting
-      lines = []
-
       # Header
-      message = sprintf(
+      @logger.info sprintf(
         "%3s %-15s %s",
         params[:way],
         params[:exchange],
@@ -44,24 +81,14 @@ module PushyDaemon
         )
 
       # Attributes
-      if (params[:attrs].is_a? Hash)
-        # lines.merge params[:attrs]
-        params[:attrs].each do |name, value|
-          lines << sprintf("%-15s %s", name, value)
-        end
-      end
+      @logger.debug params[:attrs] if params[:attrs].is_a?(Hash)
 
       # Body (split in lines to log them separately)
-      if params[:body] && params[:body].is_a?(Enumerable)
+      if params[:body].is_a?(Enumerable) && !params[:body].empty?
         body_json = JSON.pretty_generate(params[:body])
-        body_json.each_line do |line|
-          lines << line.rstrip
-        end
+        #puts "log? #{params[:body]} "
+        @logger.debug body_json.lines
       end
-
-      # Send the info
-      @logger.add Logger::INFO, message, lines
-      # @logger.log_info message, lines
     end
 
     # Start connexion to RabbitMQ
@@ -70,11 +97,12 @@ module PushyDaemon
         busconf[:host] && busconf[:port]
 
       info "connecting to #{busconf[:host]} port #{busconf[:port]}"
-      conn = Bunny.new host: (busconf[:host].to_s || "localhost").to_s,
+      conn = Bunny.new host: busconf[:host].to_s,
         port: busconf[:port].to_i,
         user: busconf[:user].to_s,
         pass: busconf[:pass].to_s,
-        heartbeat: :server
+        heartbeat: :server,
+        logger: @logger
       conn.start
 
       # Create channel
@@ -116,7 +144,8 @@ module PushyDaemon
         info "subscribe: bind [#{rule_topic}/#{route}] \t> #{rule_queue}"
 
         # Add row to config table
-        @table.add_row [rule_name, rule_topic, route, rule[:relay].to_s, rule[:title].to_s ]
+        # ["rule", "topic", "route", "relay", "queue", "description"]
+        @table.add_row [rule_name, rule_topic, route, rule[:relay].to_s, rule_queue, rule[:title].to_s ]
       end
 
       # Subscribe to our new queue

data/lib/pushyd/formatter.rb

@@ -0,0 +1,32 @@
+module PushyDaemon
+  class Formatter
+
+    def self.call severity, datetime, progname, messages
+      # Build common values
+      timestamp = datetime.strftime(LOG_FORMAT_TIME)
+
+      # If we have a bunch of lines, prefix them and send them together
+      if messages.is_a? Array
+        messages.map do |line|
+          sprintf LOG_FORMAT_ARRAY, timestamp, severity, progname, trimmed(line)
+        end.join
+
+      elsif messages.is_a? Hash
+        messages.map do |key, value|
+          sprintf LOG_FORMAT_HASH, timestamp, severity, progname, key, value
+        end.join
+
+      else
+        sprintf LOG_FORMAT_LINE, timestamp, severity, progname, trimmed(messages)
+
+      end
+    end
+
+  protected
+
+    def self.trimmed line
+      line.to_s.rstrip[0..LOG_TRIM_LINE].force_encoding(Encoding::UTF_8)
+    end
+
+  end
+end

data/lib/pushyd/proxy.rb

@@ -14,7 +14,7 @@ module PushyDaemon
       # Init ASCII table
       @table = Terminal::Table.new
       @table.title = "Propagation rules"
-      @table.headings = ["queue binding", "topic", "route", "relay", "title"]
+      @table.headings = ["rule", "topic", "route", "relay", "queue", "description"]
       @table.align_column(5, :right)
 
       # Start connexion to RabbitMQ and create channel
@@ -23,21 +23,21 @@ module PushyDaemon
 
       # Check config
       config_rules = Conf[:rules]
-      unless (config_rules.is_a? Enumerable) && !config_rules.empty?
+      if config_rules.nil? || !config_rules.is_a?(Hash)
         error "prepare: empty [rules] section"
-      end
-      info "found rules: #{config_rules.keys.join(', ')}"
-
-      # Subsribe for each and every rule/route
-      config_rules.each do |name, rule|
-        rule[:name] = name
-        channel_subscribe rule
-        #abort "prepare: OK"
+      else
+        info "found rules: #{config_rules.keys.join(', ')}"
+
+        # Subsribe for each and every rule/route
+        config_rules.each do |name, rule|
+          rule[:name] = name
+          channel_subscribe rule
+        end
       end
 
       # Send config table to logs
-      table_lines = @table.to_s.lines
-      info "initialized with configuration:", table_lines
+      info "proxy initialized"
+      info @table.to_s.lines
 
     rescue Bunny::TCPConnectionFailedForAllHosts => e
       error "ERROR: cannot connect to RabbitMQ hosts (#{e.inspect})"
@@ -100,17 +100,15 @@ module PushyDaemon
         error "propagate: #{e.message}"
     end
 
+  private
 
     def parse payload, content_type #, fields = []
       # Force encoding (pftop...)
-      utf8payload = payload.force_encoding('UTF-8')
+      utf8payload = payload.to_s.force_encoding('UTF-8')
 
       # Parse payload if content-type provided
       case content_type
         when "application/json"
-          # if fields = rule[:payload_extract]
-          #   data = payload_extract(payload, fields)
-          #   data_source = "extract #{fields.inspect} #{data.keys.count}k"
           return JSON.parse utf8payload
         when "text/plain"
           return utf8payload.to_s
@@ -124,5 +122,12 @@ module PushyDaemon
       return {}
     end
 
+    # NewRelic instrumentation
+    if Conf.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      add_transaction_tracer :handle_message, category: :task
+      add_transaction_tracer :propagate,      category: :task
+    end
+
   end
 end

data/lib/pushyd/shouter.rb

@@ -33,7 +33,11 @@ module PushyDaemon
       # Create exchange
       fail PushyDaemon::EndpointTopicContext unless @topic
       @exchange = @channel.topic(@topic, durable: true, persistent: true)
-      info "initialized with parameters:", { topic: @topic, period: @period, keys: @keys }
+
+      # Send shouter info to logs
+      shouter_info = { topic: @topic, period: @period, keys: @keys }
+      info "shouter initialized"
+      info shouter_info
 
     rescue Bunny::TCPConnectionFailedForAllHosts => e
       error "ERROR: cannot connect to RabbitMQ hosts (#{e.inspect})"
@@ -74,7 +78,11 @@ module PushyDaemon
       routing_key = keys.unshift(exchange_name).join('.')
 
       # Announce shout
-      message way: WAY_OUT, exchange: exchange_name, key: routing_key, body: nil, attrs: {}
+      message way: WAY_OUT,
+        exchange: exchange_name,
+        key: routing_key,
+        body: body,
+        attrs: {}
 
       # Publish
       @exchange.publish(body.to_json,
@@ -85,5 +93,12 @@ module PushyDaemon
         )
     end
 
+    # NewRelic instrumentation
+    if Conf.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      add_transaction_tracer :channel_shout,  category: :task
+      add_transaction_tracer :shout,          category: :task
+    end
+
   end
 end

data/lib/pushyd.rb

@@ -4,12 +4,12 @@ require "yaml"
 require "json"
 require "thread"
 require "singleton"
-# require "newrelic_rpm"
+require "newrelic_rpm"
 
 # Project libs
 require_relative "pushyd/conf"
 require_relative "pushyd/constants"
-require_relative "pushyd/pushy_logger"
+require_relative "pushyd/formatter"
 require_relative "pushyd/endpoint"
 require_relative "pushyd/proxy"
 require_relative "pushyd/shouter"

data/pushyd.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   # Project version
-  spec.version                      = "0.3.4"
+  spec.version                      = "0.4.0"
 
   # Project description
   spec.name                         = "pushyd"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pushyd
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-09 00:00:00.000000000 Z
+date: 2016-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -198,8 +198,8 @@ files:
 - lib/pushyd/constants.rb
 - lib/pushyd/daemon.rb
 - lib/pushyd/endpoint.rb
+- lib/pushyd/formatter.rb
 - lib/pushyd/proxy.rb
-- lib/pushyd/pushy_logger.rb
 - lib/pushyd/shouter.rb
 - pushyd.gemspec
 homepage: http://github.com/bmedici/pushyd

data/lib/pushyd/pushy_logger.rb

@@ -1,61 +0,0 @@
-require "logger"
-
-class PushyLogger < Logger
-
-  def initialize logfile, rotation = nil
-    # Call my parent's initializer
-    super
-
-    # And the formatter
-    self.formatter = proc do |severity, datetime, _progname, messages|
-      # Build common line prefix
-      prefix = "%s %s\t" % [
-        datetime.strftime(LOG_FORMAT_TIME),
-        severity      ]
-
-      # If we have a bunch of lines, prefix them and send them together
-      if messages.is_a? Array
-        messages.map { |line| prefix + line + LOG_NEWLINE}.join
-      else
-        prefix + messages.to_s + LOG_NEWLINE
-      end
-    end
-  end
-
-  def add level, message, lines = {}
-    level ||= Logger::DEBUG
-
-    if lines.is_a? Hash
-      output = build_from_hash lines
-    elsif lines.is_a? Array
-      output = build_from_array lines
-    else
-      output = []
-    end
-
-    # Prepend plain message to output
-    output.unshift message.force_encoding(Encoding::UTF_8)
-
-    # Send all this to logger
-    super level, output
-  end
-
-protected
-
-  def trim_line line
-    line.to_s.rstrip.strip[0..LOG_TRIM_LINE]
-  end
-
-  def build_from_array lines
-    lines.map do |value|
-      "#{LOG_INDENT}#{trim_line(value)}"
-    end
-  end
-
-  def build_from_hash lines
-    lines.map do |name, value|
-      "#{LOG_INDENT}#{name}: #{trim_line(value)}"
-    end
-  end
-
-end