logstash-input-jms 3.0.6-java → 3.1.0-java

data/lib/logstash/inputs/jms.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/inputs/base"
 require "logstash/inputs/threadable"
+require 'java'
 require "logstash/namespace"
 
 # Read events from a Jms Broker. Supports both Jms Queues and Topics.
@@ -34,6 +35,13 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   config :include_header, :validate => :boolean, :default => true
   # Include JMS Message Properties Field values in the event
   config :include_properties, :validate => :boolean, :default => true
+
+  # List of headers to skip from the event if headers are included
+  config :skip_headers, :validate => :array, :default => []
+
+  # List of properties to skip from the event if properties are included
+  config :skip_properties, :validate => :array, :default => []
+
   # Include JMS Message Body in the event
   # Supports TextMessage, MapMessage and ByteMessage
   # If the JMS Message is a TextMessage or ByteMessage, then the value will be in the "message" field of the event
@@ -50,7 +58,7 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   config :use_jms_timestamp, :validate => :boolean, :default => false
 
   # Choose an implementation of the run block. Value can be either consumer, async or thread
-  config :runner, :validate => [ "consumer", "async", "thread" ], :default => "consumer"
+  config :runner, :deprecated => true
 
   # Set the selector to use to get messages off the queue or topic
   config :selector, :validate => :string
@@ -59,13 +67,18 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   config :timeout, :validate => :number, :default => 60
 
   # Polling interval in seconds.
-  # This is the time sleeping between asks to a consumed Queue.
-  # This parameter has non influence in the case of a subcribed Topic.
   config :interval, :validate => :number, :default => 10
 
   # If pub-sub (topic) style should be used.
   config :pub_sub, :validate => :boolean, :default => false
 
+  # Durable subscriber settings.
+  # By default the `durable_subscriber_name` will be set to the topic, and `durable_subscriber_client_id` will be set
+  # to 'Logstash'
+  config :durable_subscriber, :validate => :boolean, :default => false
+  config :durable_subscriber_client_id, :validate => :string, :required => false
+  config :durable_subscriber_name, :validate => :string, :required => false
+
   # Name of the destination queue or topic to use.
   config :destination, :validate => :string, :required => true
 
@@ -88,7 +101,7 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # Username to connect to JMS provider with
   config :username, :validate => :string
   # Password to use when connecting to the JMS provider
-  config :password, :validate => :string
+  config :password, :validate => :password
   # Url to use when connecting to the JMS provider
   config :broker_url, :validate => :string
 
@@ -98,6 +111,18 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # contains details on how to connect to JNDI server
   config :jndi_context, :validate => :hash
 
+  # System properties
+  config :system_properties, :validate => :hash
+
+  # Factory settings
+  config :factory_settings, :validate => :hash
+
+  config :keystore, :validate => :path
+  config :keystore_password, :validate => :password
+  config :truststore, :validate => :path
+  config :truststore_password, :validate => :password
+
+
   # :yaml_file, :factory and :jndi_name are mutually exclusive, both cannot be supplied at the
   # same time. The priority order is :yaml_file, then :jndi_name, then :factory
   #
@@ -109,34 +134,125 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   public
   def register
     require "jms"
-    @connection = nil
 
-    if @yaml_file
-      @jms_config = YAML.load_file(@yaml_file)[@yaml_section]
+    check_config
+    load_ssl_properties
+    load_system_properties if @system_properties
+    @jms_config = jms_config
+
+    @logger.debug("JMS Config being used ", :context => obfuscate_jms_config(@jms_config))
+  end # def register
+
+  def obfuscate_jms_config(config)
+    config.each_with_object({}) { |(k, v), h| h[k] = obfuscatable?(k) ? 'xxxxx' : v }
+  end
+
+  def obfuscatable?(setting)
+    [:password, :keystore_password, :truststore_password].include?(setting)
+  end
+
+  def jms_config
+    return jms_config_from_yaml(@yaml_file, @yaml_section) if @yaml_file
+    return jms_config_from_jndi if @jndi_name
+    jms_config_from_configuration
+  end
 
-    elsif @jndi_name
-      @jms_config = {
-        :require_jars => @require_jars,
-        :jndi_name => @jndi_name,
-        :jndi_context => @jndi_context}
 
-    elsif @factory
-      @jms_config = {
+  def jms_config_from_configuration
+    config = {
         :require_jars => @require_jars,
         :factory => @factory,
         :username => @username,
-        :password => @password,
         :broker_url => @broker_url,
         :url => @broker_url # "broker_url" is named "url" with Oracle AQ
-      }
+    }
+
+    config[:password] = @password.value unless @password.nil?
+    correct_factory_hash(config, @factory_settings) unless @factory_settings.nil?
+    config
+  end
+
+  def correct_factory_hash(original, value)
+    if hash.is_a?(String)
+      return true if value.downcase == "true"
+      return false if value.downcase == "false"
     end
 
-    @logger.debug("JMS Config being used", :context => @jms_config)
+    if value.is_a?(Hash)
+      value.each { |key, value| original[key.to_sym] = correct_factory_hash({}, value) }
+      return original
+    end
+    value
+  end
 
-  end # def register
+  def jms_config_from_jndi
+    {
+        :require_jars => @require_jars,
+        :jndi_name => @jndi_name,
+        :jndi_context => @jndi_context
+    }
+  end
+
+  def jms_config_from_yaml(file, section)
+    YAML.load_file(file)[section]
+  end
+
+  def load_ssl_properties
+    java.lang.System.setProperty("javax.net.ssl.keyStore", @keystore) if @keystore
+    java.lang.System.setProperty("javax.net.ssl.keyStorePassword", @keystore_password.value) if @keystore_password
+    java.lang.System.setProperty("javax.net.ssl.trustStore", @truststore) if @truststore
+    java.lang.System.setProperty("javax.net.ssl.trustStorePassword", @truststore_password.value) if @truststore_password
+  end
+
+  def load_system_properties
+    @system_properties.each { |k,v| java.lang.System.set_property(k,v.to_s) }
+  end
+
+  def check_config
+    check_durable_subscription_config
+    raise(LogStash::ConfigurationError, "Threads cannot be > 1 if pub_sub is set") if @threads > 1 && @pub_sub
+  end
+
+  def check_durable_subscription_config
+    return unless @durable_subscriber
+    raise(LogStash::ConfigurationError, "pub_sub must be true if durable_subscriber is set") unless @pub_sub
+    @durable_subscriber_client_id ||= 'Logstash'
+    @durable_subscriber_name ||= destination
+  end
+
+  def run(output_queue)
+    begin
+      connection = JMS::Connection.new(@jms_config)
+      connection.client_id = @durable_subscriber_client_id if @durable_subscriber_client_id
+      session = connection.create_session(@jms_config)
+      connection.start
+      params = {:timeout => @timeout * 1000, :selector => @selector}
+      subscriber = subscriber(session, params)
+      until stop?
+        # This will read from the queue/topic until :timeout is breached, or messages are available whichever comes
+        # first.
+        subscriber.each({:timeout => @interval * 1000}) do |message|
+          queue_event(message, output_queue)
+          break if stop?
+        end
+      end
+    rescue => e
+      logger.warn("JMS Consumer Died", error_hash(e))
+      unless stop?
+        sleep(5)
+        subscriber && subscriber.close
+        session && session.close
+        connection && connection.close
+        retry
+      end
+    ensure
+      subscriber && subscriber.close
+      session && session.close
+      connection && connection.close
+    end
+  end # def run_consumer
 
 
-  private
   def queue_event(msg, output_queue)
     begin
       if @include_body
@@ -146,13 +262,13 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
             event.set(field.to_s, value) # TODO(claveau): needs codec.decode or converter.convert ?
           end
         elsif msg.java_kind_of?(JMS::TextMessage) || msg.java_kind_of?(JMS::BytesMessage)
-          if !msg.to_s.nil?
+          unless msg.to_s.nil?
             @codec.decode(msg.to_s) do |event_message|
               event = event_message
             end
           end
         else
-          @logger.error( "Unknown data type #{msg.data.class.to_s} in Message" )
+          @logger.error( "Unsupported message type #{msg.data.class.to_s}" )
         end
       end
 
@@ -164,14 +280,14 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
       end
 
       if @include_header
-        msg.attributes.each do |field, value|
-          event.set(field.to_s, value)
+        msg.attributes && msg.attributes.each do |field, value|
+          event.set(field.to_s, value) unless @skip_headers.include?(field.to_s)
         end
       end
 
       if @include_properties
-        msg.properties.each do |field, value|
-          event.set(field.to_s, value)
+        msg.properties && msg.properties.each do |field, value|
+          event.set(field.to_s, value) unless @skip_properties.include?(field.to_s)
         end
       end
 
@@ -180,102 +296,55 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
 
     rescue => e # parse or event creation error
       @logger.error("Failed to create event", :message => msg, :exception => e,
-                    :backtrace => e.backtrace);
+                    :backtrace => e.backtrace)
     end
   end
 
-  # Consume all available messages on the queue
-  # sleeps some time, then consume again
-  private
-  def run_consumer(output_queue)
-    JMS::Connection.session(@jms_config) do |session|
-      destination_key = @pub_sub ? :topic_name : :queue_name
-      while !stop?
-        session.consume(destination_key => @destination, :timeout=>@timeout, :selector => @selector, :buffered_message => @oracle_aq_buffered_messages) do |message|
-          queue_event message, output_queue
-          break if stop?
-        end
-        sleep @interval
-      end
-    end
-  rescue => e
-    @logger.warn("JMS Consumer died", :exception => e, :backtrace => e.backtrace)
-    sleep(10)
-    retry unless stop?
-  end # def run_consumer
-
-  # Consume all available messages on the queue through a listener
-  private
-  def run_thread(output_queue)
-    connection = JMS::Connection.new(@jms_config)
-    connection.on_exception do |jms_exception|
-      @logger.warn("JMS Exception has occurred: #{jms_exception}")
-    end
 
+  def subscriber(session, params)
     destination_key = @pub_sub ? :topic_name : :queue_name
-    connection.on_message(destination_key => @destination, :selector => @selector) do |message|
-      queue_event message, output_queue
-    end
-    connection.start
-    while !stop?
-      @logger.debug("JMS Thread sleeping ...")
-      sleep @interval
-    end
-  rescue => e
-    @logger.warn("JMS Consumer died", :exception => e, :backtrace => e.backtrace)
-    sleep(10)
-    retry unless stop?
-  end # def run_thread
-
-  # Consume all available messages on the queue through a listener
-  private
-  def run_async(output_queue)
-    JMS::Connection.start(@jms_config) do |connection|
-      # Define exception listener
-      # The problem here is that we do not handle any exception
-      connection.on_exception do |jms_exception|
-        @logger.warn("JMS Exception has occurred: #{jms_exception}")
-        raise jms_exception
-      end
-      # Define Asynchronous code block to be called every time a message is received
-      destination_key = @pub_sub ? :topic_name : :queue_name
-      connection.on_message(destination_key => @destination, :selector => @selector) do |message|
-        queue_event message, output_queue
-      end
-      # Since the on_message handler above is in a separate thread the thread needs
-      # to do some other work. It will just sleep for 10 seconds.
-      while !stop?
-        @logger.debug("JMS Thread sleeping ...")
-        sleep @interval
-      end
-    end
-  rescue => e
-    @logger.warn("JMS Consumer died", :exception => e, :backtrace => e.backtrace)
-    sleep(10)
-    retry unless stop?
-  end # def run_async
+    params[destination_key] = @destination
+    queue_or_topic = session.create_destination(params)
+    @durable_subscriber ? durable_subscriber(session, queue_or_topic, params) :
+                          regular_subscriber(session, queue_or_topic, params)
+  end
 
-  public
-  def run(output_queue)
-    case @runner
-    when "consumer" then
-      run_consumer(output_queue)
-    when "async" then
-      run_async(output_queue)
-    when "thread" then
-      run_thread(output_queue)
-    end
-  end # def run
 
-  public
-  def close
-    @logger.info("Closing JMS connection")
-    @connection.close rescue nil
-  end # def close
+  def durable_subscriber(session, queue_or_topic, params)
+    params[:selector]  ? session.create_durable_subscriber(queue_or_topic, @durable_subscriber_name, params[:selector], false) :
+                         session.create_durable_subscriber(queue_or_topic, @durable_subscriber_name)
+  end
 
-  public
-  def stop
-    @logger.info("Stopping JMS consumer")
-    @connection.stop rescue nil
-  end # def stop
+  def regular_subscriber(session, queue_or_topic, params)
+    params[:selector] ? session.create_consumer(queue_or_topic, params[:selector]) :
+                        session.create_consumer(queue_or_topic)
+  end
+
+  def error_hash(e)
+    error_hash = {:exception => e.class.name, :exception_message => e.message, :backtrace => e.backtrace}
+    root_cause = get_root_cause(e)
+    error_hash[:root_cause] = root_cause unless root_cause.nil?
+    error_hash
+  end
+
+  # JMS Exceptions can contain chains of Exceptions, making it difficult to determine the root cause of an error
+  # without knowing the actual root cause behind the problem.
+  # This method protects against Java Exceptions where the cause methods loop. If there is a cause loop, the last
+  # cause exception before the loop is detected will be returned, along with an entry in the root_cause hash indicating
+  # that an exception loop was detected. This will mean that the root cause may not be the actual root cause of the
+  # problem, and further investigation is required
+  def get_root_cause(e)
+    return nil unless e.respond_to?(:get_cause) && !e.get_cause.nil?
+    cause = e
+    slow_pointer = e
+    # Use a slow pointer to avoid cause loops in Java Exceptions
+    move_slow = false
+    until (next_cause = cause.get_cause).nil?
+      cause = next_cause
+      return {:exception => cause.class.name, :exception_message => cause.message, :exception_loop => true } if cause == slow_pointer
+      slow_pointer = slow_pointer.cause if move_slow
+      move_slow = !move_slow
+    end
+    {:exception => cause.class.name, :exception_message => cause.message }
+  end
 end # class LogStash::Inputs::Jms

data/logstash-input-jms.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-jms'
-  s.version         = '3.0.6'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from a Jms Broker"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/fixtures/activemq_plaintext.xml

@@ -0,0 +1,132 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<!-- START SNIPPET: example -->
+<beans
+  xmlns="http://www.springframework.org/schema/beans"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+  http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
+
+    <!-- Allows us to use system properties as variables in this configuration file -->
+    <bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+        <property name="locations">
+            <value>file:${activemq.conf}/credentials.properties</value>
+        </property>
+    </bean>
+
+   <!-- Allows accessing the server log -->
+    <bean id="logQuery" class="io.fabric8.insight.log.log4j.Log4jLogQuery"
+          lazy-init="false" scope="singleton"
+          init-method="start" destroy-method="stop">
+    </bean>
+
+    <!--
+        The <broker> element is used to configure the ActiveMQ broker.
+    -->
+    <broker xmlns="http://activemq.apache.org/schema/core" brokerName="localhost" dataDirectory="${activemq.data}">
+
+        <destinationPolicy>
+            <policyMap>
+              <policyEntries>
+                <policyEntry topic=">" >
+                    <!-- The constantPendingMessageLimitStrategy is used to prevent
+                         slow topic consumers to block producers and affect other consumers
+                         by limiting the number of messages that are retained
+                         For more information, see:
+
+                         http://activemq.apache.org/slow-consumer-handling.html
+
+                    -->
+                  <pendingMessageLimitStrategy>
+                    <constantPendingMessageLimitStrategy limit="1000"/>
+                  </pendingMessageLimitStrategy>
+                </policyEntry>
+              </policyEntries>
+            </policyMap>
+        </destinationPolicy>
+
+
+        <!--
+            The managementContext is used to configure how ActiveMQ is exposed in
+            JMX. By default, ActiveMQ uses the MBean server that is started by
+            the JVM. For more information, see:
+
+            http://activemq.apache.org/jmx.html
+        -->
+        <managementContext>
+            <managementContext createConnector="false"/>
+        </managementContext>
+
+        <!--
+            Configure message persistence for the broker. The default persistence
+            mechanism is the KahaDB store (identified by the kahaDB tag).
+            For more information, see:
+
+            http://activemq.apache.org/persistence.html
+        -->
+        <persistenceAdapter>
+            <kahaDB directory="${activemq.data}/kahadb"/>
+        </persistenceAdapter>
+
+
+          <!--
+            The systemUsage controls the maximum amount of space the broker will
+            use before disabling caching and/or slowing down producers. For more information, see:
+            http://activemq.apache.org/producer-flow-control.html
+          -->
+          <systemUsage>
+            <systemUsage>
+                <memoryUsage>
+                    <memoryUsage percentOfJvmHeap="70" />
+                </memoryUsage>
+                <storeUsage>
+                    <storeUsage limit="100 gb"/>
+                </storeUsage>
+                <tempUsage>
+                    <tempUsage limit="50 gb"/>
+                </tempUsage>
+            </systemUsage>
+        </systemUsage>
+
+        <!--
+            The transport connectors expose ActiveMQ over a given protocol to
+            clients and other brokers. For more information, see:
+
+            http://activemq.apache.org/configuring-transports.html
+        -->
+        <transportConnectors>
+            <!-- DOS protection, limit concurrent connections to 1000 and frame size to 100MB -->
+            <transportConnector name="openwire" uri="tcp://0.0.0.0:61616?maximumConnections=1000&amp;wireFormat.maxFrameSize=104857600"/>
+        </transportConnectors>
+
+        <!-- destroy the spring context on shutdown to stop jetty -->
+        <shutdownHooks>
+            <bean xmlns="http://www.springframework.org/schema/beans" class="org.apache.activemq.hooks.SpringContextHook" />
+        </shutdownHooks>
+
+    </broker>
+
+    <!--
+        Enable web consoles, REST and Ajax APIs and demos
+        The web consoles requires by default login, you can disable this in the jetty.xml file
+
+        Take a look at ${ACTIVEMQ_HOME}/conf/jetty.xml for more details
+    -->
+    <import resource="jetty.xml"/>
+
+</beans>
+<!-- END SNIPPET: example -->