logstash-input-jms 3.1.0-java → 3.2.1-java

data/lib/logstash/inputs/jms.rb

@@ -4,13 +4,18 @@ require "logstash/inputs/threadable"
 require 'java'
 require "logstash/namespace"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+
 # Read events from a Jms Broker. Supports both Jms Queues and Topics.
 #
 # For more information about Jms, see <http://docs.oracle.com/javaee/6/tutorial/doc/bncdq.html>
 # For more information about the Ruby Gem used, see <http://github.com/reidmorrison/jruby-jms>
 # Here is a config example to pull from a queue:
 #  jms {
-#     include_header => false
+#     include_headers => false
 #     include_properties => false
 #     include_body => true
 #     use_jms_timestamp => false
@@ -23,6 +28,12 @@ require "logstash/namespace"
 #
 #
 class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "jms"
 
   # A JMS message has three parts :
@@ -32,7 +43,8 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # You can tell the input plugin which parts should be included in the event produced by Logstash
   #
   # Include JMS Message Header Field values in the event
-  config :include_header, :validate => :boolean, :default => true
+  config :include_header, :validate => :boolean, :deprecated => "Set 'include_headers => ...' instead"
+  config :include_headers, :validate => :boolean, :default => true
   # Include JMS Message Properties Field values in the event
   config :include_properties, :validate => :boolean, :default => true
 
@@ -48,12 +60,12 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # If the JMS Message is a MapMessage, then all the key/value pairs will be added in the Hashmap of the event
   # StreamMessage and ObjectMessage are not supported
 
-  # Receive Oracle AQ buffered messages. 
+  # Receive Oracle AQ buffered messages.
   # In this mode persistent Oracle AQ JMS messages will not be received.
   config :oracle_aq_buffered_messages, :validate => :boolean, :default => false
 
   config :include_body, :validate => :boolean, :default => true
-  
+
   # Convert the JMSTimestamp header field to the @timestamp value of the event
   config :use_jms_timestamp, :validate => :boolean, :default => false
 
@@ -72,7 +84,7 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # If pub-sub (topic) style should be used.
   config :pub_sub, :validate => :boolean, :default => false
 
-  # Durable subscriber settings.
+  # Durable message_consumer settings.
   # By default the `durable_subscriber_name` will be set to the topic, and `durable_subscriber_client_id` will be set
   # to 'Logstash'
   config :durable_subscriber, :validate => :boolean, :default => false
@@ -122,6 +134,15 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   config :truststore, :validate => :path
   config :truststore_password, :validate => :password
 
+  # Defines a target field for placing fields.
+  # If this setting is omitted, data gets stored at the root (top level) of the event.
+  # The target is only relevant while decoding data into a new event.
+  #
+  # NOTE: this is only relevant for map messages; byte[] and string use the codec!
+  config :target, :validate => :field_reference
+
+  config :headers_target, :validate => :field_reference # ECS default: [@metadata][input][jms][headers]
+  config :properties_target, :validate => :field_reference # ECS default: [@metadata][input][jms][properties]
 
   # :yaml_file, :factory and :jndi_name are mutually exclusive, both cannot be supplied at the
   # same time. The priority order is :yaml_file, then :jndi_name, then :factory
@@ -132,10 +153,25 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # For some known examples, see: [Example jms.yml](https://github.com/reidmorrison/jruby-jms/blob/master/examples/jms.yml)
 
   public
+
+  def initialize(*params)
+    super
+
+    if ecs_compatibility != :disabled # set ECS target defaults
+      @headers_target = '[@metadata][input][jms][headers]' unless original_params.include?('headers_target')
+      @properties_target = '[@metadata][input][jms][properties]' unless original_params.include?('properties_target')
+    end
+
+    @headers_setter = event_setter_for(@headers_target)
+    @properties_setter = event_setter_for(@properties_target)
+
+    @headers_mapper = ecs_compatibility == :disabled ? LegacyHeadersMapper::INSTANCE : HeadersMapper::INSTANCE
+  end
+
   def register
     require "jms"
 
-    check_config
+    check_config!
     load_ssl_properties
     load_system_properties if @system_properties
     @jms_config = jms_config
@@ -208,7 +244,15 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     @system_properties.each { |k,v| java.lang.System.set_property(k,v.to_s) }
   end
 
-  def check_config
+  def check_config!
+    if original_params.include?('include_header')
+      if original_params.include?('include_headers')
+        raise(LogStash::ConfigurationError, "Both `include_headers => #{include_headers}` and `include_header => #{include_header}`" +
+                                            " options are specified, please only set one")
+      end
+      @include_headers = include_header # only `include_header => ...` was set
+    end
+
     check_durable_subscription_config
     raise(LogStash::ConfigurationError, "Threads cannot be > 1 if pub_sub is set") if @threads > 1 && @pub_sub
   end
@@ -229,9 +273,9 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
       params = {:timeout => @timeout * 1000, :selector => @selector}
       subscriber = subscriber(session, params)
       until stop?
-        # This will read from the queue/topic until :timeout is breached, or messages are available whichever comes
-        # first.
-        subscriber.each({:timeout => @interval * 1000}) do |message|
+        # read from the queue/topic until :timeout is reached, or a message is available
+        # (whichever comes first)
+        do_receive_message(subscriber, timeout: @interval * 1000) do |message|
           queue_event(message, output_queue)
           break if stop?
         end
@@ -252,43 +296,35 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     end
   end # def run_consumer
 
-
   def queue_event(msg, output_queue)
     begin
       if @include_body
         if msg.java_kind_of?(JMS::MapMessage)
-          event = LogStash::Event.new
-          msg.data.each do |field, value|
-            event.set(field.to_s, value) # TODO(claveau): needs codec.decode or converter.convert ?
-          end
+          event = process_map_message(msg)
         elsif msg.java_kind_of?(JMS::TextMessage) || msg.java_kind_of?(JMS::BytesMessage)
-          unless msg.to_s.nil?
-            @codec.decode(msg.to_s) do |event_message|
-              event = event_message
-            end
-          end
+          event = decode_message(msg)
         else
           @logger.error( "Unsupported message type #{msg.data.class.to_s}" )
         end
       end
 
-      event ||= LogStash::Event.new
+      event ||= event_factory.new_event
 
       # Here, we can use the JMS Enqueue timestamp as the @timestamp
       if @use_jms_timestamp && msg.jms_timestamp
         event.set("@timestamp", LogStash::Timestamp.at(msg.jms_timestamp / 1000, (msg.jms_timestamp % 1000) * 1000))
       end
 
-      if @include_header
-        msg.attributes && msg.attributes.each do |field, value|
-          event.set(field.to_s, value) unless @skip_headers.include?(field.to_s)
-        end
+      if @include_headers
+        headers = map_headers(msg)
+        @skip_headers.each { |key| headers.delete(key) }
+        @headers_setter.call(event, headers)
       end
 
       if @include_properties
-        msg.properties && msg.properties.each do |field, value|
-          event.set(field.to_s, value) unless @skip_properties.include?(field.to_s)
-        end
+        properties = to_string_keyed_hash(msg.properties)
+        @skip_properties.each { |key| properties.delete(key) }
+        @properties_setter.call(event, properties)
       end
 
       decorate(event)
@@ -300,6 +336,63 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     end
   end
 
+  # Loop through messages received and yield them.
+  #
+  # NOTE: a simplified replacement for JMS::MessageConsumer#each and #get (extensions).
+  #
+  # @param message_consumer [javax.jms.MessageConsumer]
+  # @param timeout in milliseconds
+  def do_receive_message(message_consumer, timeout: 0)
+    # Receive messages according to timeout
+    while true
+      case timeout
+      when  0 # Return immediately if no message is available
+        message = message_consumer.receiveNoWait()
+      when -1 # Wait forever
+        message = message_consumer.receive()
+      else # Wait for x milli-seconds for a message to be received from the broker
+        message = message_consumer.receive(timeout)
+      end
+      break unless message
+      yield message
+    end
+  end
+
+  def to_string_keyed_hash(hash)
+    hash.inject({}) { |h, (key, val)| h[key.to_s] = val; h }
+  end
+  private :to_string_keyed_hash
+
+  # @param msg [JMS::MapMessage]
+  # @return [LogStash::Event]
+  def process_map_message(msg)
+    data = to_string_keyed_hash(msg.data)
+    do_target_check_once_and_get_event_factory.new_event(data)
+  end
+
+  def do_target_check_once_and_get_event_factory
+    @target_checked ||= begin
+                          do_target_check
+                          targeted_event_factory
+                        end
+  end
+
+  TARGET_NOT_SET_MESSAGE = LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck::TARGET_NOT_SET_MESSAGE
+
+  def do_target_check
+    return true unless target.nil?
+    return nil if ecs_compatibility == :disabled
+    logger.info(TARGET_NOT_SET_MESSAGE) # target isn't set in ECS mode
+  end
+
+  # @param msg [JMS::TextMessage, JMS::BytesMessage]
+  # @return [LogStash::Event, nil]
+  def decode_message(msg)
+    text = msg.to_s # javax.jms.TextMessage#getText (e.g. JSON payload)
+    event = nil
+    @codec.decode(text) { |e| event = e } unless text.nil?
+    event
+  end
 
   def subscriber(session, params)
     destination_key = @pub_sub ? :topic_name : :queue_name
@@ -347,4 +440,102 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     end
     {:exception => cause.class.name, :exception_message => cause.message }
   end
+
+  private
+
+  def normalize_field_ref(target)
+    # so we can later event.set("#{target}[#{name}]", ...)
+    target.match?(/\A[^\[\]]+\z/) ? "[#{target}]" : target
+  end
+
+  def event_setter_for(target)
+    if target.nil? || target.empty?
+      TOP_LEVEL_EVENT_SETTER
+    else
+      TargetEventSetter.new normalize_field_ref(target)
+    end
+  end
+
+  TOP_LEVEL_EVENT_SETTER = lambda { |event, data| data.each { |key, val| event.set(key, val) } }
+  private_constant :TOP_LEVEL_EVENT_SETTER
+
+  class TargetEventSetter
+
+    def initialize(target)
+      @target = target
+    end
+
+    def call(event, data)
+      data.each { |key, val| event.set("#{@target}[#{key}]", val) }
+    end
+
+  end
+  private_constant :TargetEventSetter
+
+  def map_headers(msg)
+    @headers_mapper.call(msg)
+  end
+
+  # Maps JMS headers names
+  # (similar to JMS gem's Message#attributes extension)
+  class HeadersMapper
+
+    # @param msg [javax.jms.Message]
+    # @return [Hash]
+    def call(msg)
+      map = {
+        'jms_message_id' => msg.getJMSMessageID, # String
+        'jms_timestamp' => msg.getJMSTimestamp, # long
+        'jms_expiration' => msg.getJMSExpiration, # long
+        'jms_priority' => msg.getJMSPriority, # int (0-9)
+        'jms_type' => msg.getJMSType, # String
+        'jms_redelivered' => msg.getJMSRedelivered, # boolean
+      }
+
+      correlation_id = msg.getJMSCorrelationID # String
+      map['jms_correlation_id'] = correlation_id unless correlation_id.nil?
+
+      delivery_mode = jms_delivery_mode(msg)
+      map['jms_delivery_mode'] = delivery_mode unless delivery_mode.nil?
+
+      destination = msg.getJMSDestination # javax.jms.Destination
+      map['jms_destination'] = destination.to_string unless destination.nil?
+
+      reply_to = msg.getJMSReplyTo # javax.jms.Destination
+      map['jms_reply_to'] = reply_to.to_string unless reply_to.nil?
+
+      map
+    end
+
+    private
+
+    def jms_delivery_mode(msg)
+      case msg.getJMSDeliveryMode
+      when javax.jms.DeliveryMode::PERSISTENT
+        'persistent'
+      when javax.jms.DeliveryMode::NON_PERSISTENT
+        'non_persistent'
+      else
+        nil
+      end
+    end
+
+    INSTANCE = self.new
+
+  end
+
+  # For plugin compatibility due the use of JMS gem's Message#attributes.
+  class LegacyHeadersMapper < HeadersMapper
+
+    def call(msg)
+      map = super(msg)
+      delivery_mode = jms_delivery_mode(msg)
+      map['jms_delivery_mode_sym'] = delivery_mode ? delivery_mode.to_sym : nil
+      map
+    end
+
+    INSTANCE = self.new
+
+  end
+
 end # class LogStash::Inputs::Jms

data/logstash-input-jms.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-jms'
-  s.version         = '3.1.0'
+  s.version         = '3.2.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from a Jms Broker"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -19,15 +19,18 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
+  s.platform = RUBY_PLATFORM
+
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-json', '~> 3.0'
   s.add_runtime_dependency 'logstash-codec-plain', '~> 3.0'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
+
+  s.add_runtime_dependency "jruby-jms", ">= 1.2.0" #(Apache 2.0 license)
   s.add_runtime_dependency 'semantic_logger', '< 4.0.0'
 
-  if RUBY_PLATFORM == 'java'
-    s.platform = RUBY_PLATFORM
-    s.add_runtime_dependency "jruby-jms", ">= 1.2.0" #(Apache 2.0 license)
-  end
   s.add_development_dependency 'logstash-devutils'
 end