pushyd 0.9.4 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f6680e8b9c8639d2230c50187affeda2816b658
-  data.tar.gz: c896cb6178136eb5293a8cf0922b06364bb82f27
+  metadata.gz: f28f30eb8e374a96c94d9db2e3211cb96e130805
+  data.tar.gz: 2b3f74b4cf520b66847c1626c44e3257e6310a49
 SHA512:
-  metadata.gz: 0d9a4132921a47b2f909bd915213c9bf48606769ba8bd0d854da8c6936fc982e9bb5fb6c9e072be7ad63a0a3fc4172e0eeffc1fbc65cff8e61ecc9f53716d673
-  data.tar.gz: 9ecfa2313ed02c2df6c8285b4da4b9adc5905ef10200aad5bb62ec30272fdb71a9b3808c59308de94178848a33e3fe4041897c0556fecbfa32310c56675a3805
+  metadata.gz: 20bb1cc397e0d112988e0f8988645a553eefb19c288c81a49d531e4fc644607a45a160c3db0ff77c76a2e2e13666d877a44072e585baaf4553120ab6b56def2d
+  data.tar.gz: 167aa4659b654a46195671d00d46d6a205b19fb528aac492d6c0b8cc12ad4b8580b02975b9365eee082780785e574e8601dd26a01f8c386629a42a3f69ad9b92

data/Gemfile.lock

@@ -1,14 +1,15 @@
 PATH
   remote: .
   specs:
-    pushyd (0.9.4)
+    pushyd (0.20.0)
       api-auth
-      bmc-daemon-lib (~> 0.2)
+      bmc-daemon-lib (~> 0.3.14)
       bunny (~> 2.3)
       daemons
       json
       newrelic_rpm
       rest-client (~> 1.8)
+      rollbar
       terminal-table
 
 GEM
@@ -16,21 +17,21 @@ GEM
   specs:
     addressable (2.4.0)
     amq-protocol (2.0.1)
-    api-auth (2.0.0)
+    api-auth (2.0.1)
     ast (2.3.0)
-    bmc-daemon-lib (0.2.0)
+    bmc-daemon-lib (0.3.14)
       chamber (~> 2.9)
-    bunny (2.5.0)
+    bunny (2.5.1)
       amq-protocol (>= 2.0.1)
-    chamber (2.9.0)
+    chamber (2.9.1)
       hashie (~> 3.3)
       thor (~> 0.19.1)
-    daemons (1.2.3)
+    daemons (1.2.4)
     diff-lcs (1.2.5)
-    domain_name (0.5.20160615)
+    domain_name (0.5.20160826)
       unf (>= 0.0.5, < 1.0.0)
     hashie (3.4.4)
-    http (2.0.2)
+    http (2.0.3)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 1.0.1)
@@ -39,10 +40,11 @@ GEM
       domain_name (~> 0.5)
     http-form_data (1.0.1)
     http_parser.rb (0.6.0)
-    json (2.0.1)
-    mime-types (2.99.2)
+    json (2.0.2)
+    mime-types (2.99.3)
+    multi_json (1.12.1)
     netrc (0.11.0)
-    newrelic_rpm (3.16.0.318)
+    newrelic_rpm (3.16.2.321)
     parser (2.3.1.2)
       ast (~> 2.2)
     powerpack (0.1.1)
@@ -52,11 +54,13 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
+    rollbar (2.12.0)
+      multi_json
     rspec (3.5.0)
       rspec-core (~> 3.5.0)
       rspec-expectations (~> 3.5.0)
       rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.1)
+    rspec-core (3.5.3)
       rspec-support (~> 3.5.0)
     rspec-expectations (3.5.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -65,19 +69,20 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.5.0)
     rspec-support (3.5.0)
-    rubocop (0.41.2)
+    rubocop (0.42.0)
       parser (>= 2.3.1.1, < 3.0)
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
-    terminal-table (1.6.0)
+    terminal-table (1.7.2)
+      unicode-display_width (~> 1.1.1)
     thor (0.19.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.2)
-    unicode-display_width (1.1.0)
+    unicode-display_width (1.1.1)
 
 PLATFORMS
   ruby
@@ -91,4 +96,4 @@ DEPENDENCIES
   rubocop
 
 BUNDLED WITH
-   1.11.2
+   1.12.5

data/bin/pushyd

@@ -17,6 +17,7 @@ begin
   # Defaults
   cmd_config = nil
   cmd_logfile = nil
+  cmd_dump = nil
 
   # Init Chamber-based configuration from Gemspec
   Conf.init File.dirname(__FILE__) + "/../"
@@ -25,10 +26,11 @@ begin
   # Parse options and check compliance
   parser = OptionParser.new do |opts|
     opts.banner = "Usage: #{File.basename $PROGRAM_NAME} [options] start|stop"
-    opts.on("-l", "--log LOGFILE")           { |path| cmd_logfile = File.expand_path(path.to_s)}
-    opts.on("-c", "--config CONFIGFILE")     { |path| cmd_config = File.expand_path(path.to_s)}
-    opts.on("-e", "--environment ENV")       { |env| Conf.app_env = env }
-    opts.on("",   "--dev")                   { Conf.app_env = "development" }
+    opts.on("-l", "--log LOGFILE")                                  { |path| cmd_logfile = File.expand_path(path.to_s)}
+    opts.on("-c", "--config CONFIGFILE")                            { |path| cmd_config = File.expand_path(path.to_s)}
+    opts.on("-e", "--environment ENV")                              { |env| Conf.app_env = env }
+    opts.on("",   "--dev")                                          { Conf.app_env = "development" }
+    opts.on("",   "--dump", "Dump config as seen by the process")   { |value| cmd_dump = true }
   end
   parser.order!(ARGV)
 
@@ -55,8 +57,12 @@ puts "Config files    \t #{Conf.files}"
 puts "Started at      \t #{Conf.app_started}"
 puts "Loging to file  \t #{Conf[:log][:file]}" if Conf[:log].is_a? Enumerable
 puts "Process name    \t #{Conf.generate(:process_name)}"
-puts
-puts Conf.dump
+puts "Newrelic        \t #{Conf.feature?(:newrelic) || '-'}"
+puts "Rollbar         \t #{Conf.feature?(:rollbar) || '-'}"
+if cmd_dump
+  puts
+  puts Conf.dump
+end
 
 # Run daemon
 run_options = {

data/defaults.yml

@@ -1,10 +1,14 @@
 # common defaults
 broker: amqp://guest:guest@localhost:5672/%2F
+
 logs:
-  path: '/tmp/'
-  file: pushyd.log
-  newrelic: newrelic.log
-  level: debug
+  path:     "/tmp/"
+  level:    debug
+  default:  "pushyd-default.log"
+  shouter:  "pushyd-shouter.log"
+  consumer: "pushyd-consumer.log"
+  # newrelic: "pushyd-newrelic.log"
+  # rollbar:  "pushyd-rollbar.log"
 
 shout:
   topic: pushyd

data/lib/pushyd/constants.rb

@@ -1,29 +1,25 @@
 # Constants: global
 MSG_SEND                = "SEND"
 MSG_RECV                = "RECV"
-WAY_PROP                = "PROP"
+MSG_RLAY                = "RLAY"
 
-# Constants: proxy
-PROXY_MESSAGE_MAX       = 1
-PROXY_USE_ACK           = false
+# Constants: AMQP protocol
+AMQP_HEARTBEAT_INTERVAL = 30
+AMQP_RECOVERY_INTERVAL  = 5
+AMQP_PREFETCH           = 3
+AMQP_MANUAL_ACK         = false
 
 # Constants: shouter
 SHOUTER_SENTAT_DECIMALS = 6
 
 # Constants: logger
-LOG_ROTATION            = "daily"
-
 LOG_HEADER_TIME         = "%Y-%m-%d %H:%M:%S"
 LOG_HEADER_FORMAT       = "%s \t%d\t%-8s %-15s "
-LOG_MESSAGE_TRIM        = 200
+LOG_MESSAGE_TRIM        = 250
 LOG_MESSAGE_TEXT        = "%s%s"
 LOG_MESSAGE_ARRAY       = "%s     %s"
 LOG_MESSAGE_HASH        = "%s     %-20s %s\n"
 
 # Constants: logger app-specific prefix
-LOG_PREFIX_FORMAT       = nil
-
-# Constants: AMQP protocol
-AMQP_HEARTBEAT_INTERVAL = 30
-AMQP_RECOVERY_INTERVAL  = 5
-AMQP_PREFETCH           = 2
+#LOG_PREFIX_FORMAT       = "(%-12s) (%-12s)"
+LOG_PREFIX_FORMAT       = "%-20s "

data/lib/pushyd/consumer.rb

@@ -0,0 +1,144 @@
+module PushyDaemon
+  # class ShouterResponseError       < StandardError; end
+  # class ShouterChannelClosed       < StandardError; end
+  # class ShouterPreconditionFailed  < StandardError; end
+  # class ShouterInterrupted         < StandardError; end
+  class ConsumerError              < StandardError; end
+  class ConsumerRuleMissing        < StandardError; end
+
+  class Consumer < BmcDaemonLib::MqConsumer
+    #include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    include Shared::HmacSignature
+    attr_accessor :logger
+
+    def initialize(conn, rule_name, rule)
+      # Init
+      @queue = nil
+      @conn = conn
+      @rule = rule
+      @rule_name = rule_name
+
+      # Prepare logger
+      @logger = BmcDaemonLib::LoggerPool.instance.get :consumer
+
+      # Create channel, prefetch only one message at a time
+      @channel = @conn.create_channel
+      @channel.prefetch(AMQP_PREFETCH)
+
+      # OK
+      log_info "Consumer initialized"
+    end
+
+  protected
+
+    def log_prefix
+      [@rule_name]
+    end
+
+    # Handle the reception of a message on a queue
+    def handle_message context, metadata, delivery_info, message = {}
+      # Prepare data
+      headers = metadata.headers || {}
+
+      # Relay data if needed
+      handle_relay context, message, headers
+
+      # Handle errors and acknowledgments
+      # log_debug "handle_message : channel[#{@channel.inspect}]"
+      rescue StandardError => e
+        log_error "handle_message: unknown: #{e.message}, #{e.inspect}", e.backtrace
+        channel_ackit(message[:tag], false)
+      else
+        channel_ackit(message[:tag], true)
+    end
+
+  private
+
+    def handle_relay context, message, headers
+      # Check we have a valid @rule
+      raise ConsumerRuleMissing unless @rule.is_a? Hash
+
+      # Check if we need to relay anything
+      unless @rule[:relay]
+        log_debug "handle_relay: no [relay] URL"
+        return
+      end
+
+      # Prepare stuff
+      relay_auth  = @rule[:auth].to_s
+      relay_url   = URI(@rule[:relay]).to_s
+      request_id  = identifier(6)
+      request_prefix = "handle_relay [#{request_id}] "
+
+      # Build payload
+      request_infos = {
+        topic:    message[:topic],
+        route:    message[:rkey],
+        sent_at:  headers['sent_at'],
+        sent_by:  headers['sent_by'],
+        context:  context,
+        data:     message[:data],
+        }
+      request_body = JSON.pretty_generate(request_infos)
+
+      # Build request headers
+      headers = {
+        content_type: :json,
+        accept: :json,
+        user_agent: BmcDaemonLib::Conf.generate(:user_agent),
+        }
+
+      # Compute: payload MD5, HMAC signature
+      headers_md5 headers, request_body
+      headers_sign headers, @rule[:sign], [:date]
+
+      # Build final request
+      request = RestClient::Request.new url: relay_url,
+        method: :post,
+        payload: request_body,
+        headers: headers
+
+      # Execute request
+      log_message MSG_RLAY, request_id, relay_url, request_infos, request.processed_headers
+      response = request.execute
+
+      # Handle exceptions
+      rescue RestClient::ExceptionWithResponse, URI::InvalidURIError, RestClient::InternalServerError => e
+        log_error "#{request_prefix} rest-client exception: #{e.message}"
+      rescue ApiAuth::ApiAuthError, ApiAuth::UnknownHTTPRequest => e
+        log_error "#{request_prefix} api-auth: #{e.message}"
+      rescue Errno::ECONNREFUSED => e
+        log_error "#{request_prefix} connection refused: #{e.message}"
+      rescue StandardError => e
+        log_error "#{request_prefix} unknown: #{e.message}, #{e.inspect}", e.backtrace
+      else
+        log_info "#{request_prefix} received [#{response.body}]"
+    end
+
+    def channel_ackit tag, success=true
+      # log_debug "channel_ackit[#{channel}.#{tag}] #{@channel.inspect}"
+    #   if success
+    #     log_debug "channel_ackit[#{@channel.id}.#{tag}]: ACK"
+    #     @channel.ack(tag)
+    #   else
+    #     log_debug "channel_ackit[#{@channel.id}.#{tag}]: NACK"
+    #     @channel.nack(tag)
+    #   end
+
+    # rescue Bunny::ChannelAlreadyClosed => ex
+    #   error "channel_ackit[#{@channel.id}.#{tag}]: exception: ChannelAlreadyClosed"
+
+    # rescue StandardError => ex
+    #   log_debug "channel_ackit[#{@channel.id}.#{tag}]: exception: #{ex.inspect}"
+    #   # fail PushyDaemon::EndpointSubscribeError, "unhandled (#{e.inspect})"
+
+    # else
+    #   log_debug "channel_ackit[#{@channel.id}.#{tag}]: done"
+    end
+
+    # NewRelic instrumentation
+    #add_transaction_tracer :receive,        category: :task
+    #add_transaction_tracer :propagate,      category: :task
+
+  end
+end

data/lib/pushyd/daemon.rb

@@ -2,20 +2,12 @@ module PushyDaemon
   class Daemon
 
     def self.run
-      # Create a new proxy
-      p = Proxy.new
-
-      # Dump config table
-      puts p.table.to_s
-
-      # Create a new shouter
-      s = Shouter.new
-
-      # Start shout loop
-      s.shout
+      # Create a new proxy, and dump its configuration
+      Proxy.new
 
       # Backup infinite loop in case shout does nothing
       loop do
+        sleep 1
       end
 
     rescue EndpointConnectionError, ShouterInterrupted => e

data/lib/pushyd/initialize.rb

@@ -0,0 +1,3 @@
+# Init Rollbar and Newrelic
+Conf.prepare_newrelic
+Conf.prepare_rollbar

data/lib/pushyd/proxy.rb

@@ -1,19 +1,19 @@
+require "bunny"
 require 'api_auth'
 require 'rest_client'
 require 'terminal-table'
 
 module PushyDaemon
-  class Proxy < Endpoint
-    include Shared::HmacSignature
-    include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+  class Proxy < BmcDaemonLib::MqEndpoint
+    #include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
 
     # Class options
     attr_accessor :table
 
     def initialize
       # Init
-      super
-      @exchanges = {}
+      @shouters = []
+      @consumers = []
 
       # Init ASCII table
       @table = Terminal::Table.new
@@ -21,155 +21,87 @@ module PushyDaemon
       @table.headings = ["rule", "topic", "route", "relay", "created queue", "description"]
       @table.align_column(5, :right)
 
-      # Start connexion to RabbitMQ and create channel
-      @channel = connect_channel Conf[:broker]
-      log_info "channel connected"
+      # Prepare logger
+      @logger = BmcDaemonLib::LoggerPool.instance.get
 
-      # Check config
-      config_rules = Conf[:rules]
-      if config_rules.nil? || !config_rules.is_a?(Hash)
-        log_error "prepare: empty [rules] section"
-      else
-        log_info "found rules: #{config_rules.keys.join(', ')}"
-
-        # Subsribe for each and every rule/route
-        config_rules.each do |name, rule|
-          rule[:name] = name
-          channel_subscribe rule
-        end
-      end
+      # Start connexion to RabbitMQ
+      @conn = connect_to BmcDaemonLib::Conf[:broker]
+      log_info "Proxy connected"
+
+      # Check config and subscribe rules
+      create_consumers
+
+      # Create a new shouter, and start its loop
+      create_shouter
 
       # Send config table to logs
-      log_info "proxy initialized", @table.to_s.lines
+      log_info "Proxy initialized", @table.to_s.lines
     end
 
   protected
 
-  private
+    def log_prefix
+      [nil]
+      # [self.class.name.split('::').last, :proxy]
+    end
 
-    def extract_delay msg_headers
-      return unless msg_headers['sent_at']
+    def create_shouter
+      # Get config
+      config_shouter = BmcDaemonLib::Conf[:shout]
 
-      # Extract sent_at header
-      sent_at = Time.iso8601(msg_headers['sent_at']) rescue nil
-      # log_info "sent_at     : #{sent_at.to_f}"
-      # log_info "timenow     : #{Time.now.to_f}"
+      # Create the shouter
+      shouter = Shouter.new(@conn, config_shouter)
+      @shouters << shouter
 
-      # Compute delay
-      return ((Time.now - sent_at)*1000).round(2)
+      # Now make it loop
+      shouter.start_loop
     end
 
+    def create_consumers
+      # Get config
+      config_rules = BmcDaemonLib::Conf[:rules]
+      log_info "create_consumers: #{config_rules.keys.join(', ')}"
 
-    # Handle the reception of a message on a queue
-    def handle_message rule, delivery_info, metadata, payload
-      # Prepare data
-      rule_name = rule[:name]
-      msg_exchange = delivery_info.exchange
-      msg_rkey = delivery_info.routing_key.force_encoding('UTF-8')
-      msg_headers = metadata.headers || {}
-
-      # Compute delay and size
-      delay = extract_delay(msg_headers)
-      size = format_bytes(payload.bytesize, "B")
-
-      # Extract payload
-      data = parse payload, metadata.content_type
-
-      # Announce match
-      log_message MSG_RECV, msg_exchange, msg_rkey, data, {
-        'matched rule' => rule_name,
-        'app-id' => metadata.app_id,
-        'content-type' => metadata.content_type,
-        'delay (ms)' => delay,
-        'body size' => size,
-        }
-
-      # Build notification payload
-      propagate_data = {
-        exchange: msg_exchange,
-        route: msg_rkey,
-        sent_at: msg_headers['sent_at'],
-        sent_by: msg_headers['sent_by'],
-        data: data,
-        }
-
-      # Propagate data if needed
-      propagate rule, propagate_data
-    end
+      if config_rules.nil? || !config_rules.is_a?(Hash)
+        log_error "prepare: no rules"
+        return
+      end
 
-    def propagate rule, data
-      # Nothing more to do if no relay
-      return if rule[:relay].nil? || rule[:relay].empty?
-
-      # Prepare stuff
-      relay_auth = rule[:auth].to_s
-      relay_uri = URI(rule[:relay])
-      relay_url = relay_uri.to_s
-      id = identifier(6)
-      # log_info "propagate: user[#{relay_uri.user}] url[#{relay_url}]"
-
-      # Build POST body and log message
-      post_body = JSON.pretty_generate(data)
-      log_message WAY_PROP, id, relay_url, data
-
-      # Prepare request
-      request = RestClient::Request.new url: relay_url,
-        method: :post,
-        payload: post_body,
-        headers: {
-          content_type: :json,
-          accept: :json,
-          user_agent: Conf.generate(:user_agent),
-          }
-
-      # Compute payload MD5
-      headers_md5 request
-
-      # Compute HMAC signature
-      headers_sign request, rule['hmac-method'], rule['hmac-user'], rule['hmac-secret'], [:date]
-
-      # Send request
-      log_info "propagate: #{relay_url}", request.headers
-      response = request.execute
-
-      # Handle exceptions
-      rescue RestClient::ExceptionWithResponse, URI::InvalidURIError => e
-        log_error "propagate: rest-client: #{e.message}"
-      rescue RestClient::InternalServerError => e
-        log_error "propagate: rest-client: #{e.message}"
-      rescue ApiAuth::ApiAuthError, ApiAuth::UnknownHTTPRequest => e
-        log_error "propagate: api-auth: #{e.message}"
-      rescue Errno::ECONNREFUSED => e
-        log_error "propagate: connection refused: #{e.message}"
-      rescue StandardError => e
-        log_error "propagate: unknown: #{e.message}, #{e.inspect}", e.backtrace
-      else
-        log_info "propagate: #{response.body}"
+      # Subscribe for each and every rule/route
+      config_rules.each do |name, rule|
+        rule[:name] = name
+        create_consumer rule
+      end
     end
 
-    def parse payload, content_type #, fields = []
-      # Force encoding (pftop...)
-      utf8payload = payload.to_s.force_encoding('UTF-8')
-
-      # Parse payload if content-type provided
-      case content_type
-        when "application/json"
-          return JSON.parse utf8payload rescue nil
-        when "text/plain"
-          return utf8payload.to_s
-        else
-          return utf8payload
+    # Subscribe to interesting topic/routes and bind a listenner
+    def create_consumer rule
+      # Check information
+      rule_name   = rule[:name].to_s
+      rule_topic  = rule[:topic].to_s
+      rule_routes = rule[:routes].to_s.split(' ')
+      rule_queue  = sprintf('%s-%s', BmcDaemonLib::Conf.app_name, rule_name.gsub('_', '-'))
+
+      fail PushyDaemon::EndpointSubscribeContext, "rule [#{rule_name}] lacking topic" unless rule_topic
+      fail PushyDaemon::EndpointSubscribeContext, "rule [#{rule_name}] lacking routes" if rule_routes.empty?
+
+      # Build a new consumer
+      consumer = Consumer.new(@conn, rule_name, rule)
+
+      # Create its own queue
+      consumer.subscribe_to_queue rule_queue, "rule:#{rule_name}"
+
+      # Bind each route to exchange
+      rule_routes.each do |route|
+        consumer.listen_to rule_topic, route
+
+        # Add row to config table
+        @table.add_row [rule_name, rule_topic, route, rule[:relay].to_s, rule_queue, rule[:title].to_s ]
       end
 
-    # Handle body parse errors
-    rescue Encoding::UndefinedConversionError => e
-      log_error "parse: JSON PARSE ERROR: #{e.inspect}"
-      return {}
+      # Return it
+      @consumers << consumer
     end
 
-    # NewRelic instrumentation
-    add_transaction_tracer :handle_message, category: :task
-    add_transaction_tracer :propagate,      category: :task
-
   end
 end

data/lib/pushyd/shouter.rb

@@ -5,63 +5,55 @@ module PushyDaemon
   class ShouterInterrupted         < StandardError; end
   class EndpointTopicContext       < StandardError; end
 
-  class Shouter < Endpoint
+  class Shouter < BmcDaemonLib::MqEndpoint
     include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    include BmcDaemonLib::LoggerHelper
 
     # Class options
     attr_accessor :table
+    attr_accessor :logger
 
-    def initialize
+    def initialize(conn, config_shout)
       # Init
-      super
-      @keys = []
+      @shouter_keys = []
+      @channel = nil
+      @exchange = nil
+
+      # Prepare logger
+      @logger = BmcDaemonLib::LoggerPool.instance.get :shouter
 
       # Check config
-      config_shout = Conf[:shout]
       unless config_shout && config_shout.any? && config_shout.is_a?(Enumerable)
         log_error "prepare: empty [shout] section"
         return
       end
 
       # Extract information
-      @keys = config_shout[:keys] if config_shout[:keys].is_a? Array
-      @topic = config_shout[:topic]
-      @period = config_shout[:period] || 0
+      @shouter_keys = config_shout[:keys] if config_shout[:keys].is_a? Array
+      @shouter_topic = config_shout[:topic]
+      @shouter_period = config_shout[:period].to_i
+      @shouter_period = 1 unless (@shouter_period > 0)
 
-      # Start connexion to RabbitMQ and create channel
-      @channel = connect_channel Conf[:broker]
-      log_info "channel connected"
+      fail PushyDaemon::EndpointTopicContext unless @shouter_topic
 
-      # Create exchange
-      fail PushyDaemon::EndpointTopicContext unless @topic
-      @exchange = @channel.topic(@topic, durable: true, persistent: true)
+      # Create channel and exchange
+      @channel = conn.create_channel
+      @exchange = @channel.topic(@shouter_topic, durable: true, persistent: true)
 
-      # Send shouter info to logs
-      shouter_info = { topic: @topic, period: @period, keys: @keys }
-      log_info "shouter initialized", shouter_info
+      # Start working, now
+      log_info "Shouter initialized, starting loop now", { topic: @shouter_topic, period: @shouter_period, keys: @shouter_keys }
     end
 
-    def shout
-      return unless @exchange
-
+    def start_loop
       # Prepare exchange
       loop do
-        # Generate key and fake id
-        if @keys.is_a?(Array) && @keys.any?
-          random_key = @keys.sample
-        else
-          random_key = "random"
-        end
-        # random_key = @keys.class
-        random_string = SecureRandom.hex
-
         # Generate payload
-        #payload = {time: Time.now.to_f, host: Conf.host}
-        payload = nil
+        payload = {time: Time.now.to_f, host: BmcDaemonLib::Conf.host}
+        # payload = nil
 
         # Shout it !
-        channel_shout [:ping, random_key, random_string], payload
-        sleep @period
+        exchange_shout @exchange, payload
+        sleep @shouter_period
       end
     rescue AMQ::Protocol::EmptyResponseError => e
       fail PushyDaemon::ShouterResponseError, "#{e.class} (#{e.inspect})"
@@ -76,35 +68,43 @@ module PushyDaemon
 
   protected
 
+    def log_prefix
+      [nil]
+      # [self.class.name.split('::').last, :shouter]
+    end
+
   private
 
-    def channel_shout keys, body = {}
-      # Prepare exchange_name and routing_key
-      exchange_name = @exchange.name
-      keys.unshift(exchange_name)
+    def exchange_shout exchange, body = {}
+      # Prepare routing_key
+      keys = []
+      keys << @shouter_topic
+      keys << SecureRandom.hex
+      keys << @shouter_keys.sample if (@shouter_keys.is_a?(Array) && @shouter_keys.any?)
       routing_key = keys.join('.')
 
       # Announce shout
-      log_message MSG_SEND, exchange_name, routing_key, body
+      log_message MSG_SEND, @shouter_topic, routing_key, body
 
       # Prepare headers
+      app_id = "#{BmcDaemonLib::Conf.app_name}/#{BmcDaemonLib::Conf.app_ver}"
       headers = {
         sent_at: DateTime.now.iso8601(SHOUTER_SENTAT_DECIMALS),
-        sent_by: Conf.app_name,
+        sent_by: app_id,
         }
 
       # Publish
-      @exchange.publish(body.to_json,
+      exchange.publish(body.to_json,
         routing_key: routing_key,
         headers: headers,
-        app_id: Conf.app_name,
+        app_id: app_id,
         content_type: "application/json",
         )
     end
 
     # NewRelic instrumentation
-    add_transaction_tracer :channel_shout,  category: :task
-    add_transaction_tracer :shout,          category: :task
+    #add_transaction_tracer :exchange_shout,  category: :task
+    # add_transaction_tracer :shout,           category: :task
 
   end
 end

data/lib/pushyd.rb

@@ -4,13 +4,18 @@ require 'bmc-daemon-lib'
 require "yaml"
 require "json"
 require "newrelic_rpm"
+require "rollbar"
+
 
 # Shared libs
 require_relative "shared/hmac_signature"
 
 # Project libs
 require_relative "pushyd/constants"
-require_relative "pushyd/endpoint"
 require_relative "pushyd/proxy"
+require_relative "pushyd/consumer"
 require_relative "pushyd/shouter"
 require_relative "pushyd/daemon"
+
+# Init
+require_relative "pushyd/initialize"

data/lib/shared/hmac_signature.rb

@@ -4,21 +4,33 @@ require 'base64'
 module Shared
   module HmacSignature
 
-    def headers_sign request, hmac_method, hmac_user, hmac_secret, names = ['date']
-      return unless hmac_user
-      unless hmac_secret && hmac_method
-        log_error "headers_sign: hmac: missing secret or method"
+    def headers_sign headers, config, names = ['date']
+      # Extract and check
+      return unless config.is_a? Hash
+      hmac_method = config[:method]
+      hmac_user   = config[:user]
+      hmac_secret = config[:secret]
+      #log_debug "headers_sign config", config
+
+      # Check params
+      unless config[:method] && config[:user] && config[:secret]
+        log_error "headers_sign: missing method/user/secret"
+        return
+      end
+
+      # Check params
+      unless config[:method] == 'hmac-kong'
+        log_error "headers_sign: only [hmac-kong] method is supported"
         return
       end
 
       # OK, lets go
-      log_info "headers_sign: before: user[#{hmac_user}] secret[#{hmac_secret}] method[#{hmac_method}]", request.headers
-      hmac_sign_kong request.headers, hmac_user, hmac_secret, names
-      log_info "headers_sign: after:", request.headers
+      hmac_sign_kong headers, config[:user], config[:secret], names
+      # log_info "headers_sign: after signing", headers
     end
 
-    def headers_md5 request
-      request.headers['Content-MD5'] = Digest::MD5.hexdigest(request.payload.to_s)
+    def headers_md5 headers, payload
+      headers['Content-MD5'] = Digest::MD5.hexdigest(payload.to_s)
     end
 
   private
@@ -33,18 +45,17 @@ module Shared
       myheaders = hmac_headers_filter headers, names
 
       # Signe string of headers
-      headers_signature = hmac_headers_hash myheaders, client_secret
-      log_debug "hmac_sign_kong #{myheaders.keys.inspect} #{headers_signature}"
+      signature = hmac_headers_hash myheaders, client_secret
+      log_debug "hmac_sign_kong signed [#{signature}] from headers #{myheaders.keys.inspect}"
 
       # Add auth header
-      # headers['Authorization'] = hmac_build_header(client_id, myheaders, headers_signature)
-      headers['test'] = "testing123"
+      headers['Authorization'] = hmac_build_header(client_id, myheaders, signature)
+      #headers['test'] = "testing123"
 
       # That's OK
       return headers
     end
 
-
     def hmac_build_header client_id, myheaders, signature
       sprintf 'hmac username="%s", algorithm="hmac-sha1", headers="%s", signature="%s"',
         client_id,

data/pushyd.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   # Project version
-  spec.version                      = "0.9.4"
+  spec.version                      = "0.20.0"
 
   # Project description
   spec.name                         = "pushyd"
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   # spec.add_development_dependency "pry"
 
   # Runtime dependencies
-  spec.add_runtime_dependency       "bmc-daemon-lib", "~> 0.2"
+  spec.add_runtime_dependency       "bmc-daemon-lib", "~> 0.3.14"
   spec.add_runtime_dependency       "daemons"
   spec.add_runtime_dependency       "json"
   spec.add_runtime_dependency       "bunny", "~> 2.3"
@@ -36,4 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency       "api-auth"
   spec.add_runtime_dependency       "terminal-table"
   spec.add_runtime_dependency       "newrelic_rpm"
+  spec.add_runtime_dependency       "rollbar"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pushyd
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.20.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-21 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.3.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.3.14
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rollbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A nice proxy listenning to a RabbitMQ bus, repeating selected messages
   in POST requests when filters match routing patterns
 email: pushyd@bmconseil.com
@@ -210,8 +224,9 @@ files:
 - defaults.yml
 - lib/pushyd.rb
 - lib/pushyd/constants.rb
+- lib/pushyd/consumer.rb
 - lib/pushyd/daemon.rb
-- lib/pushyd/endpoint.rb
+- lib/pushyd/initialize.rb
 - lib/pushyd/proxy.rb
 - lib/pushyd/shouter.rb
 - lib/shared/hmac_signature.rb

data/lib/pushyd/endpoint.rb

@@ -1,137 +0,0 @@
-require 'bunny'
-require "securerandom"
-
-module PushyDaemon
-  # Class exceptions
-  class EndpointConnexionContext    < StandardError; end
-  class EndpointConnectionError     < StandardError; end
-  class EndpointSubscribeContext    < StandardError; end
-  class EndpointSubscribeError      < StandardError; end
-
-  class Endpoint
-    include BmcDaemonLib::LoggerHelper
-    attr_reader :logger
-
-    def initialize
-      # Prepare logger
-      @logger = BmcDaemonLib::LoggerPool.instance.get :file
-
-      # Done
-      log_info "endpoint initialized"
-    end
-
-  protected
-
-    def log_message msg_way, msg_exchange, msg_key, msg_body = [], msg_attrs = {}
-      # Message header
-      info sprintf("%3s %-15s %s", msg_way, msg_exchange, msg_key)
-
-      # Body lines
-      if msg_body.is_a?(Enumerable) && !msg_body.empty?
-        body_json = JSON.pretty_generate(msg_body)
-        log_debug nil, body_json.lines
-      end
-
-      # Attributes lines
-      log_debug nil, msg_attrs if msg_attrs
-    end
-
-    # Start connexion to RabbitMQ
-    def connect_channel busconf
-      fail PushyDaemon::EndpointConnexionContext, "invalid bus host/port" unless busconf
-      info "connecting to bus", {
-        broker: busconf,
-        recover: AMQP_RECOVERY_INTERVAL,
-        heartbeat: AMQP_HEARTBEAT_INTERVAL,
-        prefetch: AMQP_PREFETCH
-        }
-      conn = Bunny.new busconf.to_s,
-        logger: @logger,
-        # heartbeat: :server,
-        automatically_recover: true,
-        network_recovery_interval: AMQP_RECOVERY_INTERVAL,
-        heartbeat_interval: AMQP_HEARTBEAT_INTERVAL
-      conn.start
-
-      # Create channel, prefetch only one message at a time
-      channel = conn.create_channel
-      channel.prefetch(AMQP_PREFETCH)
-
-    rescue Bunny::TCPConnectionFailedForAllHosts, Bunny::AuthenticationFailureError, AMQ::Protocol::EmptyResponseError  => e
-      fail PushyDaemon::EndpointConnectionError, "error connecting (#{e.class})"
-    rescue StandardError => e
-      fail PushyDaemon::EndpointConnectionError, "unknow (#{e.inspect})"
-    else
-      return channel
-    end
-
-    # Declare or return the exchange for this topic
-    def channel_exchange topic
-      @exchanges ||= {}
-      @exchanges[topic] ||= @channel.topic(topic, durable: true, persistent: true)
-    end
-
-    # Subscribe to interesting topic/routes and bind a listenner
-    def channel_subscribe rule
-      # Check information
-      rule_name = rule[:name].to_s
-      rule_topic = rule[:topic].to_s
-      rule_routes = rule[:routes].to_s.split(' ')
-      rule_queue = "#{Conf.app_name}-#{rule[:name]}"
-      fail PushyDaemon::EndpointSubscribeContext, "rule [#{rule_name}] lacking topic" unless rule_topic
-      fail PushyDaemon::EndpointSubscribeContext, "rule [#{rule_name}] lacking routes" if rule_routes.empty?
-
-      # Create queue for this rule (remove it beforehand)
-      #conn.create_channel.queue_delete(rule_queue_name)
-      queue = @channel.queue(rule_queue, auto_delete: false, durable: true)
-
-      # Bind each route from this topic-exchange
-      topic_exchange = channel_exchange(rule_topic)
-      rule_routes.each do |route|
-        # Bind exchange to queue
-        queue.bind topic_exchange, routing_key: route
-        info "subscribe: bind [#{rule_topic}/#{route}] \t> #{rule_queue}"
-
-        # Add row to config table
-        # ["rule", "topic", "route", "relay", "queue", "description"]
-        @table.add_row [rule_name, rule_topic, route, rule[:relay].to_s, rule_queue, rule[:title].to_s ]
-      end
-
-      # Subscribe to our new queue
-      queue.subscribe(block: false, manual_ack: PROXY_USE_ACK, message_max: PROXY_MESSAGE_MAX) do |delivery_info, metadata, payload|
-
-        # Handle the message
-        handle_message rule, delivery_info, metadata, payload
-
-      end
-
-    rescue Bunny::PreconditionFailed => e
-      fail PushyDaemon::EndpointSubscribeError, "PreconditionFailed: [#{rule_topic}] code(#{e.channel_close.reply_code}) message(#{e.channel_close.reply_text})"
-
-    rescue StandardError => e
-      fail PushyDaemon::EndpointSubscribeError, "unhandled (#{e.inspect})"
-
-    end
-
-    def handle_message rule, delivery_info, metadata, payload
-    end
-
-    def identifier len
-      rand(36**len).to_s(36)
-    end
-
-    def format_bytes number, unit="", decimals = 0
-      return "&Oslash;" if number.nil? || number.to_f.zero?
-
-      units = ["", "k", "M", "G", "T", "P" ]
-      index = ( Math.log(number) / Math.log(2) ).to_i / 10
-      converted = number.to_f / (1024 ** index)
-
-      truncated = converted.round(decimals)
-      return "#{truncated} #{units[index]}#{unit}"
-    end
-
-  private
-
-  end
-end