logstash-input-jms 3.1.2-java → 3.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9523dfd590735b242d04d1e13c9b024367cf864e0e34fdedcd23dfabacead46
-  data.tar.gz: afdfffb20005867db82bba2ff40557b3752dcbac1cbcd57fdfcab057b95813d2
+  metadata.gz: 2a40f29ee0db67e912d547f5fa74a74bcd08466c3f2594dd9edd5f82669ea494
+  data.tar.gz: c8fa0a986cea0a588b8b68fbd8618f24122ee42abb21ad3cf7f5e7cbb0f3484c
 SHA512:
-  metadata.gz: 78f2d4db68c4dbf16910aa6813e5d622743e2c7c8cc1cd8c91e2a54cdb31698350df79f6d6bea910ca611c863dcb0d4ff3c1bdee42c9948c00f88c05dfab426b
-  data.tar.gz: 2371457a596688f1960f8921508252d3bd78ea072b3283fa6492964a81e07cd288893b4c2379973872f9df202a492e4ad01a0d57115d2397df4a9254e8283e4c
+  metadata.gz: a121b9359ff97942888218bb1f2553f469940574e7f8421f7e910980bf37b973fd73cb2e0739a7525c0b71c5e8b4891765f70e505a798d1d9263f0bedbb13eee
+  data.tar.gz: d10f028310928b17f41f48cc50e8393022171d2c2fa4d25502f428f3d2bfbfc6172eee75d527d0ebb9b66f5a3790b6fe9c50fc3de5355dd71a907be33d5f9c5c

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.2.0
+ - Feat: event_factory support + targets to aid ECS [#49](https://github.com/logstash-plugins/logstash-input-jms/pull/49)
+ - Fix: when configured to add JMS headers to the event, headers whose value is not set no longer result in nil entries on the event
+ - Fix: when adding the `jms_reply_to` header to an event, a string representation is set instead of an opaque object.
+
 ## 3.1.2
  - Docs: Added additional troubleshooting information [#38](https://github.com/logstash-plugins/logstash-input-jms/pull/38)
 
@@ -45,7 +50,7 @@
  - New dependency requirements for logstash-core for the 5.0 release
 
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-jms.svg)](https://travis-ci.org/logstash-plugins/logstash-input-jms)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-jms.svg)](https://travis-ci.com/logstash-plugins/logstash-input-jms)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -33,6 +33,14 @@ JMS configurations can be done either entirely in the Logstash configuration fil
   configurations, should also use the combination of yaml file and Logstash configuration.
 
 
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+JMS data is application specific. ECS compliance for fields depends on the use case.
+The plugin includes sensible defaults that change based on <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>.
+When ECS compatibility is disabled, headers, properties, and payload are targeted at the root level of the event to maintain compatibility with legacy usage of this plugin.
+When targeting an ECS version, headers and properties target `@metadata` sub-fields unless configured otherwise in order to avoid conflict with ECS fields.
+See <<plugins-{type}s-{plugin}-headers_target>>, <<plugins-{type}s-{plugin}-properties_target>> and <<plugins-{type}s-{plugin}-target>>.
 
 ==== Sample Configuration using Logstash Configuration Only
 
@@ -60,7 +68,7 @@ The JMS plugin can also be configured using JNDI if desired.
         truststore => '/Users/logstash-user/security/truststore.jks'
         truststore_password => 'yet_another_secret'
         # Parts of the JMS message to be included <8>
-        include_header => false
+        include_headers => false
         include_properties => false
         include_body => true
         # Message selector
@@ -98,7 +106,7 @@ The JMS plugin can also be configured using JNDI if desired.
  input {
     jms {
         # Logstash Configuration Settings. <1>
-        include_header => false
+        include_headers => false
         include_properties => false
         include_body => true
         use_jms_timestamp => false
@@ -144,7 +152,7 @@ This section contains sample configurations for connecting to a JMS provider tha
  input {
     jms {
         # Logstash Configuration File Settings <1>
-        include_header => false
+        include_headers => false
         include_properties => false
         include_body => true
         use_jms_timestamp => false
@@ -205,7 +213,7 @@ This section contains sample configurations for connecting to a JMS provider tha
  input {
     jms {
         # Logstash specific configuration settings <1>
-        include_header => false
+        include_headers => false
         include_properties => false
         include_body => true
         use_jms_timestamp => false
@@ -379,10 +387,13 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-durable_subscriber>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-durable_subscriber_client_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-durable_subscriber_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-factory>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-factory_settings>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-headers_target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-include_body>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-include_header>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-include_headers>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-include_properties>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jndi_context>> |<<hash,hash>>|No
@@ -391,6 +402,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-oracle_aq_buffered_messages>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-properties_target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-pub_sub>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-require_jars>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-runner>> |<<string,string>>|__Deprecated__
@@ -398,6 +410,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-skip_headers>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-skip_properties>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-system_properties>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-threads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
@@ -430,6 +443,8 @@ Url to use when connecting to the JMS provider. This is only relevant for non-JN
 
 Name of the destination queue or topic to use.
 
+TIP: If the destination setting doesn't appear to be working properly, try this format: `<JMS module name>!<JMS queue name>`.
+
 [id="plugins-{type}s-{plugin}-durable_subscriber"]
 ===== `durable_subscriber`
 
@@ -466,6 +481,21 @@ This represents the value of the client ID for a durable subscribtion, and is on
 This represents the value of the subscriber name for a durable subscribtion, and is only used if `durable_subscriber`
 is set to `true`. Please consult your JMS Provider documentation for constraints and requirements for this setting.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event)
+    ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, JMS specific properties)
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+The value of this setting affects the _default_ value of <<plugins-{type}s-{plugin}-headers_target>> and
+<<plugins-{type}s-{plugin}-properties_target>>.
+
 [id="plugins-{type}s-{plugin}-factory"]
 ===== `factory`
 
@@ -486,6 +516,16 @@ Hash of implementation specific configuration values to set on the connection fa
  `exclusive_consumer => true` would call `setExclusiveConsumer(true)` on the supplied connection factory.
  See your JMS provider documentation for implementation specific details.
 
+[id="plugins-{type}s-{plugin}-headers_target"]
+===== `headers_target`
+
+  * Value type is <<string,string>>
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: no default value for this setting
+    ** ECS Compatibility enabled: `"[@metadata][input][jms][headers]"
+
+The name of the field under which JMS headers will be added, if <<plugins-{type}s-{plugin}-include_headers>> is set.
+
 [id="plugins-{type}s-{plugin}-include_body"]
 ===== `include_body`
 
@@ -493,29 +533,54 @@ Hash of implementation specific configuration values to set on the connection fa
   * Default value is `true`
 
 Include JMS Message Body in the event.
-Supports TextMessage, MapMessage and ByteMessage.
+Supports TextMessage, MapMessage and BytesMessage.
 
-If the JMS Message is a TextMessage or ByteMessage, then the value will be in
+If the JMS Message is a TextMessage or BytesMessage, then the value will be in
 the "message" field of the event. If the JMS Message is a MapMessage, then all
-the key/value pairs will be added in the Hashmap of the event.
+the key/value pairs will be added at the top-level of the event by default.
+To avoid pollution of the top-level namespace, when receiving a MapMessage, use the <<plugins-{type}s-{plugin}-target>>.
 
 StreamMessage and ObjectMessage are not supported.
 
 [id="plugins-{type}s-{plugin}-include_header"]
 ===== `include_header`
 
+  * Value type is <<boolean,boolean>>
+  * This option is deprecated
+
+Note: This option is deprecated and it will be removed in the next major version of Logstash.
+Use `include_headers` instead.
+
+[id="plugins-{type}s-{plugin}-include_headers"]
+===== `include_headers`
+
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
 A JMS message has three parts:
 
-* Message Headers (required)
-* Message Properties (optional)
-* Message Bodies (optional)
+  * Message Headers (required)
+  * Message Properties (optional)
+  * Message Body (optional)
 
 You can tell the input plugin which parts should be included in the event produced by Logstash.
 
-Include JMS Message Header Field values in the event.
+Include standard JMS message header field values in the event.
+Example headers:
+[source,ruby]
+-----
+    {
+        "jms_message_id" => "ID:amqhost-39547-1636977297920-71:1:1:1:1",
+        "jms_timestamp" => 1636977329102,
+        "jms_expiration" => 0,
+        "jms_delivery_mode" => "persistent",
+        "jms_redelivered" => false,
+        "jms_destination" => "topic://41ad5342149901ad",
+        "jms_priority" => 4,
+        "jms_type" => "sample",
+        "jms_correlation_id" => "28d975cb-14ff-4285-841e-05ef1e0a7ab2"
+    }
+-----
 
 [id="plugins-{type}s-{plugin}-include_properties"]
 ===== `include_properties`
@@ -588,6 +653,16 @@ Only for use with Oracle AQ
 
 Password to use when connecting to the JMS provider.
 
+[id="plugins-{type}s-{plugin}-properties_target"]
+===== `properties_target`
+
+  * Value type is <<string,string>>
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: no default value for this setting
+    ** ECS Compatibility enabled: `"[@metadata][input][jms][properties]"
+
+The name of the field under which JMS properties will be added, if <<plugins-{type}s-{plugin}-include_properties>> is set.
+
 [id="plugins-{type}s-{plugin}-pub_sub"]
 ===== `pub_sub`
 
@@ -648,6 +723,17 @@ If `include_properties` is set, a list of properties to skip processing on can b
 
 Any System properties that the JMS provider requires can be set either in a Hash here, or in `jvm.options`
 
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the field to assign MapMessage data into.
+If not specified data will be stored in the root of the event.
+
+NOTE: For TextMessage and BytesMessage the `target` has no effect. Use the codec setting instead
+e.g. `codec => json { target => "[jms]" }`.
 
 [id="plugins-{type}s-{plugin}-threads"]
 ===== `threads`
@@ -655,7 +741,7 @@ Any System properties that the JMS provider requires can be set either in a Hash
   * Value type is <<number,number>>
   * Default value is `1`
 
-* Note that if pub_sub is set to true, this value *must* be 1. A configuration error will be thrown otherwise
+NOTE: If pub_sub is set to `true`, this value *must* be `1`. A configuration error will be thrown otherwise!
 
 [id="plugins-{type}s-{plugin}-timeout"]
 ===== `timeout`
@@ -722,4 +808,4 @@ For some known examples, see https://github.com/reidmorrison/jruby-jms/blob/mast
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/inputs/jms.rb

@@ -4,13 +4,18 @@ require "logstash/inputs/threadable"
 require 'java'
 require "logstash/namespace"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+
 # Read events from a Jms Broker. Supports both Jms Queues and Topics.
 #
 # For more information about Jms, see <http://docs.oracle.com/javaee/6/tutorial/doc/bncdq.html>
 # For more information about the Ruby Gem used, see <http://github.com/reidmorrison/jruby-jms>
 # Here is a config example to pull from a queue:
 #  jms {
-#     include_header => false
+#     include_headers => false
 #     include_properties => false
 #     include_body => true
 #     use_jms_timestamp => false
@@ -23,6 +28,12 @@ require "logstash/namespace"
 #
 #
 class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "jms"
 
   # A JMS message has three parts :
@@ -32,7 +43,8 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # You can tell the input plugin which parts should be included in the event produced by Logstash
   #
   # Include JMS Message Header Field values in the event
-  config :include_header, :validate => :boolean, :default => true
+  config :include_header, :validate => :boolean, :deprecated => "Set 'include_headers => ...' instead"
+  config :include_headers, :validate => :boolean, :default => true
   # Include JMS Message Properties Field values in the event
   config :include_properties, :validate => :boolean, :default => true
 
@@ -48,12 +60,12 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # If the JMS Message is a MapMessage, then all the key/value pairs will be added in the Hashmap of the event
   # StreamMessage and ObjectMessage are not supported
 
-  # Receive Oracle AQ buffered messages. 
+  # Receive Oracle AQ buffered messages.
   # In this mode persistent Oracle AQ JMS messages will not be received.
   config :oracle_aq_buffered_messages, :validate => :boolean, :default => false
 
   config :include_body, :validate => :boolean, :default => true
-  
+
   # Convert the JMSTimestamp header field to the @timestamp value of the event
   config :use_jms_timestamp, :validate => :boolean, :default => false
 
@@ -122,6 +134,15 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   config :truststore, :validate => :path
   config :truststore_password, :validate => :password
 
+  # Defines a target field for placing fields.
+  # If this setting is omitted, data gets stored at the root (top level) of the event.
+  # The target is only relevant while decoding data into a new event.
+  #
+  # NOTE: this is only relevant for map messages; byte[] and string use the codec!
+  config :target, :validate => :field_reference
+
+  config :headers_target, :validate => :field_reference # ECS default: [@metadata][input][jms][headers]
+  config :properties_target, :validate => :field_reference # ECS default: [@metadata][input][jms][properties]
 
   # :yaml_file, :factory and :jndi_name are mutually exclusive, both cannot be supplied at the
   # same time. The priority order is :yaml_file, then :jndi_name, then :factory
@@ -132,10 +153,25 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
   # For some known examples, see: [Example jms.yml](https://github.com/reidmorrison/jruby-jms/blob/master/examples/jms.yml)
 
   public
+
+  def initialize(*params)
+    super
+
+    if ecs_compatibility != :disabled # set ECS target defaults
+      @headers_target = '[@metadata][input][jms][headers]' unless original_params.include?('headers_target')
+      @properties_target = '[@metadata][input][jms][properties]' unless original_params.include?('properties_target')
+    end
+
+    @headers_setter = event_setter_for(@headers_target)
+    @properties_setter = event_setter_for(@properties_target)
+
+    @headers_mapper = ecs_compatibility == :disabled ? LegacyHeadersMapper::INSTANCE : HeadersMapper::INSTANCE
+  end
+
   def register
     require "jms"
 
-    check_config
+    check_config!
     load_ssl_properties
     load_system_properties if @system_properties
     @jms_config = jms_config
@@ -208,7 +244,15 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     @system_properties.each { |k,v| java.lang.System.set_property(k,v.to_s) }
   end
 
-  def check_config
+  def check_config!
+    if original_params.include?('include_header')
+      if original_params.include?('include_headers')
+        raise(LogStash::ConfigurationError, "Both `include_headers => #{include_headers}` and `include_header => #{include_header}`" +
+                                            " options are specified, please only set one")
+      end
+      @include_headers = include_header # only `include_header => ...` was set
+    end
+
     check_durable_subscription_config
     raise(LogStash::ConfigurationError, "Threads cannot be > 1 if pub_sub is set") if @threads > 1 && @pub_sub
   end
@@ -252,43 +296,35 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     end
   end # def run_consumer
 
-
   def queue_event(msg, output_queue)
     begin
       if @include_body
         if msg.java_kind_of?(JMS::MapMessage)
-          event = LogStash::Event.new
-          msg.data.each do |field, value|
-            event.set(field.to_s, value) # TODO(claveau): needs codec.decode or converter.convert ?
-          end
+          event = process_map_message(msg)
         elsif msg.java_kind_of?(JMS::TextMessage) || msg.java_kind_of?(JMS::BytesMessage)
-          unless msg.to_s.nil?
-            @codec.decode(msg.to_s) do |event_message|
-              event = event_message
-            end
-          end
+          event = decode_message(msg)
         else
           @logger.error( "Unsupported message type #{msg.data.class.to_s}" )
         end
       end
 
-      event ||= LogStash::Event.new
+      event ||= event_factory.new_event
 
       # Here, we can use the JMS Enqueue timestamp as the @timestamp
       if @use_jms_timestamp && msg.jms_timestamp
         event.set("@timestamp", LogStash::Timestamp.at(msg.jms_timestamp / 1000, (msg.jms_timestamp % 1000) * 1000))
       end
 
-      if @include_header
-        msg.attributes && msg.attributes.each do |field, value|
-          event.set(field.to_s, value) unless @skip_headers.include?(field.to_s)
-        end
+      if @include_headers
+        headers = map_headers(msg)
+        @skip_headers.each { |key| headers.delete(key) }
+        @headers_setter.call(event, headers)
       end
 
       if @include_properties
-        msg.properties && msg.properties.each do |field, value|
-          event.set(field.to_s, value) unless @skip_properties.include?(field.to_s)
-        end
+        properties = to_string_keyed_hash(msg.properties)
+        @skip_properties.each { |key| properties.delete(key) }
+        @properties_setter.call(event, properties)
       end
 
       decorate(event)
@@ -300,6 +336,41 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     end
   end
 
+  def to_string_keyed_hash(hash)
+    hash.inject({}) { |h, (key, val)| h[key.to_s] = val; h }
+  end
+  private :to_string_keyed_hash
+
+  # @param msg [JMS::MapMessage]
+  # @return [LogStash::Event]
+  def process_map_message(msg)
+    data = to_string_keyed_hash(msg.data)
+    do_target_check_once_and_get_event_factory.new_event(data)
+  end
+
+  def do_target_check_once_and_get_event_factory
+    @target_checked ||= begin
+                          do_target_check
+                          targeted_event_factory
+                        end
+  end
+
+  TARGET_NOT_SET_MESSAGE = LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck::TARGET_NOT_SET_MESSAGE
+
+  def do_target_check
+    return true unless target.nil?
+    return nil if ecs_compatibility == :disabled
+    logger.info(TARGET_NOT_SET_MESSAGE) # target isn't set in ECS mode
+  end
+
+  # @param msg [JMS::TextMessage, JMS::BytesMessage]
+  # @return [LogStash::Event, nil]
+  def decode_message(msg)
+    text = msg.to_s # javax.jms.TextMessage#getText (e.g. JSON payload)
+    event = nil
+    @codec.decode(text) { |e| event = e } unless text.nil?
+    event
+  end
 
   def subscriber(session, params)
     destination_key = @pub_sub ? :topic_name : :queue_name
@@ -347,4 +418,102 @@ class LogStash::Inputs::Jms < LogStash::Inputs::Threadable
     end
     {:exception => cause.class.name, :exception_message => cause.message }
   end
+
+  private
+
+  def normalize_field_ref(target)
+    # so we can later event.set("#{target}[#{name}]", ...)
+    target.match?(/\A[^\[\]]+\z/) ? "[#{target}]" : target
+  end
+
+  def event_setter_for(target)
+    if target.nil? || target.empty?
+      TOP_LEVEL_EVENT_SETTER
+    else
+      TargetEventSetter.new normalize_field_ref(target)
+    end
+  end
+
+  TOP_LEVEL_EVENT_SETTER = lambda { |event, data| data.each { |key, val| event.set(key, val) } }
+  private_constant :TOP_LEVEL_EVENT_SETTER
+
+  class TargetEventSetter
+
+    def initialize(target)
+      @target = target
+    end
+
+    def call(event, data)
+      data.each { |key, val| event.set("#{@target}[#{key}]", val) }
+    end
+
+  end
+  private_constant :TargetEventSetter
+
+  def map_headers(msg)
+    @headers_mapper.call(msg)
+  end
+
+  # Maps JMS headers names
+  # (similar to JMS gem's Message#attributes extension)
+  class HeadersMapper
+
+    # @param msg [javax.jms.Message]
+    # @return [Hash]
+    def call(msg)
+      map = {
+        'jms_message_id' => msg.getJMSMessageID, # String
+        'jms_timestamp' => msg.getJMSTimestamp, # long
+        'jms_expiration' => msg.getJMSExpiration, # long
+        'jms_priority' => msg.getJMSPriority, # int (0-9)
+        'jms_type' => msg.getJMSType, # String
+        'jms_redelivered' => msg.getJMSRedelivered, # boolean
+      }
+
+      correlation_id = msg.getJMSCorrelationID # String
+      map['jms_correlation_id'] = correlation_id unless correlation_id.nil?
+
+      delivery_mode = jms_delivery_mode(msg)
+      map['jms_delivery_mode'] = delivery_mode unless delivery_mode.nil?
+
+      destination = msg.getJMSDestination # javax.jms.Destination
+      map['jms_destination'] = destination.to_string unless destination.nil?
+
+      reply_to = msg.getJMSReplyTo # javax.jms.Destination
+      map['jms_reply_to'] = reply_to.to_string unless reply_to.nil?
+
+      map
+    end
+
+    private
+
+    def jms_delivery_mode(msg)
+      case msg.getJMSDeliveryMode
+      when javax.jms.DeliveryMode::PERSISTENT
+        'persistent'
+      when javax.jms.DeliveryMode::NON_PERSISTENT
+        'non_persistent'
+      else
+        nil
+      end
+    end
+
+    INSTANCE = self.new
+
+  end
+
+  # For plugin compatibility due the use of JMS gem's Message#attributes.
+  class LegacyHeadersMapper < HeadersMapper
+
+    def call(msg)
+      map = super(msg)
+      delivery_mode = jms_delivery_mode(msg)
+      map['jms_delivery_mode_sym'] = delivery_mode ? delivery_mode.to_sym : nil
+      map
+    end
+
+    INSTANCE = self.new
+
+  end
+
 end # class LogStash::Inputs::Jms

data/logstash-input-jms.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-jms'
-  s.version         = '3.1.2'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from a Jms Broker"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -19,15 +19,18 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
+  s.platform = RUBY_PLATFORM
+
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-json', '~> 3.0'
   s.add_runtime_dependency 'logstash-codec-plain', '~> 3.0'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
+
+  s.add_runtime_dependency "jruby-jms", ">= 1.2.0" #(Apache 2.0 license)
   s.add_runtime_dependency 'semantic_logger', '< 4.0.0'
 
-  if RUBY_PLATFORM == 'java'
-    s.platform = RUBY_PLATFORM
-    s.add_runtime_dependency "jruby-jms", ">= 1.2.0" #(Apache 2.0 license)
-  end
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/integration/jms_spec.rb

@@ -1,11 +1,8 @@
 require_relative '../spec_helper'
 require 'logstash/inputs/jms'
-require 'jms'
-require 'json'
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 require 'securerandom'
 
-
-
 shared_examples_for "a JMS input" do
   context 'when inputting messages' do
     let (:destination) { "#{pub_sub ? 'topic' : 'queue'}://#{queue_name}"}
@@ -15,7 +12,7 @@ shared_examples_for "a JMS input" do
       let(:message) { "Hello There" }
 
       context 'when properties are skipped' do
-        let (:jms_config) { super.merge({'skip_properties' => ['this', 'that']})}
+        let (:config) { super().merge({'skip_properties' => ['this', 'that']})}
 
         it 'should skip the specified property and process other properties, headers and the message' do
           send_message do |session|
@@ -25,7 +22,7 @@ shared_examples_for "a JMS input" do
             msg.set_string_property('the_other', 'the_other_prop')
             msg
           end
-          expect(queue.first.get('message')).to eql (message)
+          expect(queue.first.get('message')).to eql(message)
           expect(queue.first.get('jms_destination')).to_not be_nil
           expect(queue.first.get('jms_timestamp')).to_not be_nil
           expect(queue.first.get('this')).to be_nil
@@ -35,7 +32,7 @@ shared_examples_for "a JMS input" do
       end
 
       context 'when using message selectors' do
-        let (:jms_config) { super.merge({'selector' => selector }) }
+        let (:config) { super().merge({'selector' => selector }) }
 
         context 'with multiple selector query parameter' do
           let (:selector) { "this = 3 OR this = 4" }
@@ -47,7 +44,7 @@ shared_examples_for "a JMS input" do
               msg.set_int_property('this', 4)
               msg
             end
-            expect(queue.first.get('message')).to eql (message)
+            expect(queue.first.get('message')).to eql(message)
             expect(queue.first.get('this')).to eql(4)
             expect(queue.first.get('that')).to eql('that_prop')
           end
@@ -73,7 +70,7 @@ shared_examples_for "a JMS input" do
               msg.set_int_property('this', 3)
               msg
             end
-            expect(queue.first.get('message')).to eql (message)
+            expect(queue.first.get('message')).to eql(message)
             expect(queue.first.get('this')).to eql(3)
             expect(queue.first.get('that')).to eql('that_prop')
           end
@@ -126,7 +123,7 @@ shared_examples_for "a JMS input" do
               msg.set_string_property('that', 'that_prop')
               msg
             end
-            expect(queue.first.get('message')).to eql (message)
+            expect(queue.first.get('message')).to eql(message)
             expect(queue.first.get('this')).to eql('this_prop')
             expect(queue.first.get('that')).to eql('that_prop')
           end
@@ -145,7 +142,8 @@ shared_examples_for "a JMS input" do
       end
 
       context 'when headers are skipped' do
-        let (:jms_config) { super.merge({'skip_headers' => ['jms_destination', 'jms_reply_to']})}
+        let (:config) { super().merge('skip_headers' => ['jms_destination', 'jms_reply_to']) }
+
         it 'should skip the specified header and process other headers, properties and the message' do
           send_message do |session|
             msg = session.message(message)
@@ -155,7 +153,7 @@ shared_examples_for "a JMS input" do
             msg.set_string_property('the_other', 'the_other_prop')
             msg
           end
-          expect(queue.first.get('message')).to eql (message)
+          expect(queue.first.get('message')).to eql(message)
           expect(queue.first.get('jms_destination')).to be_nil
           expect(queue.first.get('jms_timestamp')).to_not be_nil
           expect(queue.first.get('this')).to eq('this_prop')
@@ -164,33 +162,124 @@ shared_examples_for "a JMS input" do
         end
       end
 
-      context 'when neither header nor property is skipped ' do
-        it 'should process properties, headers and the message' do
+      context 'when include_headers => false' do
+        let (:config) { super().merge('include_headers' => 'false') }
+
+        it 'should skip all headers' do
           send_message do |session|
             msg = session.message(message)
-            msg.set_string_property('this', 'this_prop')
-            msg.set_string_property('that', 'that_prop')
-            msg.set_string_property('the_other', 'the_other_prop')
+            msg.reply_to = session.create_destination(:topic_name => SecureRandom.hex(8))
+            msg.set_string_property('some', 'property')
             msg
           end
-          expect(queue.first.get('message')).to eql (message)
-          expect(queue.first.get('jms_timestamp')).to_not be_nil
-          expect(queue.first.get('jms_destination')).to_not be_nil
-          expect(queue.first.get('this')).to eq('this_prop')
-          expect(queue.first.get('that')).to eq('that_prop')
-          expect(queue.first.get('the_other')).to eq('the_other_prop')
+          event = queue.first.to_hash_with_metadata
+          expect( event.keys.find { |name| name.start_with?('jms_') } ).to be nil
+        end
+      end
+
+      context 'when include_header => false (deprecated)' do
+        let (:config) { super().merge('include_header' => 'false') }
+
+        it 'should skip all headers' do
+          send_message do |session|
+            msg = session.message(message)
+            msg.reply_to = session.create_destination(:topic_name => SecureRandom.hex(8))
+            msg.set_string_property('some', 'property')
+            msg
+          end
+          event = queue.first.to_hash_with_metadata
+          expect( event.keys.find { |name| name.start_with?('jms_') } ).to be nil
+        end
+      end
+
+      context 'when neither header nor property is skipped', :ecs_compatibility_support do
+        ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+          let(:ecs_compatibility?) { ecs_select.active_mode != :disabled }
+
+          let (:config) { super().merge('ecs_compatibility' => ecs_select.active_mode) }
+
+          it 'should process properties, headers and the message' do
+            send_message do |session|
+              msg = session.message(message)
+              msg.set_string_property('this', 'this_prop')
+              msg.set_int_property('camelCase', 42)
+              msg.set_boolean_property('JMSFlag', true)
+              msg
+            end
+
+            event = queue.first
+
+            expect(event.get('message')).to eql(message)
+
+            # headers
+            if ecs_compatibility?
+              expect(event.include?('jms_timestamp')).to be false
+              expect(event.include?('jms_destination')).to be false
+              expect(event.get('[@metadata][input][jms][headers][jms_timestamp]')).to be_a Integer
+              expect(event.get('[@metadata][input][jms][headers][jms_destination]')).to_not be nil
+              expect(event.include?("[@metadata][input][jms][headers][jms_delivery_mode_sym]")).to be false
+              expect(event.include?("[@metadata][input][jms][headers][jms_delivery_mode]")).to be true
+            else
+              expect(event.get('jms_timestamp')).to be_a Integer
+              expect(event.get('jms_destination')).to_not be nil
+              expect(event.include?("jms_delivery_mode_sym")).to be true
+            end
+
+            # properties
+            if ecs_compatibility?
+              expect(event.include?('this')).to be false
+              expect(event.get('[@metadata][input][jms][properties][this]')).to eq 'this_prop'
+              expect(event.get('[@metadata][input][jms][properties][camelCase]')).to eq 42
+              expect(event.get('[@metadata][input][jms][properties][JMSFlag]')).to be true
+            else
+              expect(event.get('this')).to eq 'this_prop'
+              expect(event.get('camelCase')).to eq 42
+              expect(event.get('JMSFlag')).to be true
+            end
+          end
+
         end
       end
     end
 
-    context 'when the message is map message' do
-      let(:message) { {:one => 1} }
-      it 'should read the message' do
-        send_message
-        expect(queue.size).to eql 1
-        expect(queue.first.get('one')).to eql (1)
-        expect(queue.first.get("jms_destination")).to eql(destination)
+    context 'when the message is map message', :ecs_compatibility_support do
+
+      ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+        let(:ecs_compatibility?) { ecs_select.active_mode != :disabled }
+
+        let (:config) { super().merge('ecs_compatibility' => ecs_select.active_mode) }
+
+        let(:message) { {:one => 1} }
+
+        before do
+          if ecs_compatibility?
+            expect(subject.logger).to receive(:info).once.with /ECS compatibility is enabled but `target` option was not specified/i
+          end
+        end
+
+        it 'should read the message' do
+          send_message
+
+          expect(queue.size).to eql 1
+          event = queue.first
+          expect(event.get('one')).to eql 1
+
+          if ecs_compatibility?
+            expect(event.get('[@metadata][input][jms][headers][jms_destination]')).to eql(destination)
+            expect(event.get('[@metadata][input][jms][headers][jms_delivery_mode]')).to eql 'persistent'
+            expect(event.include?('[@metadata][input][jms][headers][jms_delivery_mode_sym]')).to be false
+          else
+            expect(event.get("jms_destination")).to eql(destination)
+            expect(event.get("jms_delivery_mode_sym")).to eql :persistent
+          end
+
+          send_message # should not log the ECS warning again
+        end
+
       end
+
     end
 
     context 'when the message is a bytes message' do
@@ -203,17 +292,26 @@ shared_examples_for "a JMS input" do
           jms_message
         end
         expect(queue.size).to eql 1
-        expect(queue.first.get('message')).to eql ('hello world')
+        expect(queue.first.get('message')).to eql 'hello world'
         expect(queue.first.get("jms_destination")).to eql(destination)
       end
     end
   end
 end
 
-describe "input/jms", :integration => true do
+describe LogStash::Inputs::Jms, :integration => true do
   let (:message) { "hello World" }
   let (:queue_name) { SecureRandom.hex(8)}
 
+  let (:yaml_section) { 'activemq' }
+  let (:config) {{'yaml_file' => fixture_path("jms.yml"),
+                      'yaml_section' => yaml_section,
+                      'destination' => queue_name,
+                      'pub_sub' => pub_sub,
+                      'interval' => 2}}
+
+  subject(:input) { described_class.new(config) }
+
   before :each do
     allow(input).to receive(:jms_config_from_yaml) do |yaml_file, section|
       settings = YAML.load_file(yaml_file)[section]
@@ -222,14 +320,6 @@ describe "input/jms", :integration => true do
     end
   end
 
-  let (:yaml_section) { 'activemq' }
-  let (:jms_config) {{'yaml_file' => fixture_path("jms.yml"),
-                      'yaml_section' => yaml_section,
-                      'destination' => queue_name,
-                      'pub_sub' => pub_sub,
-                      'interval' => 2}}
-  let(:input) { LogStash::Plugin.lookup("input", "jms").new(jms_config) }
-
   after :each do
     input.close unless input.nil?
   end
@@ -241,7 +331,7 @@ describe "input/jms", :integration => true do
     end
 
     context 'with pub_sub true and durable subscriber' do
-      let (:jms_config) { super.merge({'durable_subscriber' => true,
+      let (:config) { super().merge({'durable_subscriber' => true,
                            'durable_subscriber_client_id' => SecureRandom.hex(8),
                            'durable_subscriber_name' => SecureRandom.hex(8) } ) }
 
@@ -258,8 +348,8 @@ describe "input/jms", :integration => true do
 
   context 'with tls', :tls => true do
     let (:yaml_section) { 'activemq_tls' }
-    let (:jms_config) { super.merge({"keystore" => fixture_path("keystore.jks"), "keystore_password" => "changeit",
-                                     "truststore" => fixture_path("keystore.jks"), "truststore_password" => "changeit"})}
+    let (:config) { super().merge({"keystore" => fixture_path("keystore.jks"), "keystore_password" => "changeit",
+                                   "truststore" => fixture_path("keystore.jks"), "truststore_password" => "changeit"})}
 
     context 'with pub_sub true' do
       let (:pub_sub) { true }
@@ -267,7 +357,7 @@ describe "input/jms", :integration => true do
     end
 
     context 'with pub_sub true and durable subscriber' do
-      let (:jms_config) { super.merge({'durable_subscriber' => true,
+      let (:config) { super().merge({'durable_subscriber' => true,
                            'durable_subscriber_client_id' => SecureRandom.hex(8),
                            'durable_subscriber_name' => SecureRandom.hex(8) } ) }
 

data/spec/inputs/unit/jms_spec.rb

@@ -1,9 +1,9 @@
 require_relative '../spec_helper'
 require 'logstash/inputs/jms'
-require 'jms'
-require 'json'
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+require 'securerandom'
 
-describe "inputs/jms" do
+describe LogStash::Inputs::Jms do
   let (:queue_name) {SecureRandom.hex(8)}
   let (:jms_config) {{'destination' => queue_name}}
 
@@ -37,10 +37,10 @@ describe "inputs/jms" do
     context 'configuration check' do
       context 'if threads is > 1' do
           let(:thread_count) { 2 }
-          let(:jms_config) { super.merge({'threads' => thread_count})}
+          let(:jms_config) { super().merge({'threads' => thread_count})}
 
           context 'with pub_sub set to true' do
-            let(:jms_config) { super.merge({'pub_sub' => true})}
+            let(:jms_config) { super().merge({'pub_sub' => true})}
 
             it 'should raise a configuration error' do
               expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
@@ -48,7 +48,7 @@ describe "inputs/jms" do
           end
 
           context 'with pub_sub set to false' do
-            let(:jms_config) { super.merge({'pub_sub' => false})}
+            let(:jms_config) { super().merge({'pub_sub' => false})}
 
             it 'should not raise a configuration error' do
               plugin.register
@@ -58,13 +58,13 @@ describe "inputs/jms" do
       end
 
       context 'with durable_subscriber set' do
-        let(:jms_config) { super.merge({ 'pub_sub' => true,
+        let(:jms_config) { super().merge({ 'pub_sub' => true,
                                          'durable_subscriber' => true,
                                          'durable_subscriber_name' => SecureRandom.hex(8),
                                          'durable_subscriber_client_id' => SecureRandom.hex(8)})}
 
         context 'if durable_subscriber_client_id is not set' do
-          let(:jms_config) { super.tap { |h| h.delete('durable_subscriber_client_id') } }
+          let(:jms_config) { super().tap { |h| h.delete('durable_subscriber_client_id') } }
 
           it 'should set client_id to Logstash' do
             plugin.register
@@ -73,7 +73,7 @@ describe "inputs/jms" do
         end
 
         context 'if durable_subscriber_name is not set' do
-          let(:jms_config) { super.tap { |h| h.delete('durable_subscriber_name') } }
+          let(:jms_config) { super().tap { |h| h.delete('durable_subscriber_name') } }
 
           it 'should set name to the topic name' do
             plugin.register
@@ -83,7 +83,7 @@ describe "inputs/jms" do
 
 
         context 'if pub_sub is set to false' do
-          let(:jms_config) { super.merge({'pub_sub' => false})}
+          let(:jms_config) { super().merge({'pub_sub' => false})}
           it 'should raise a configuration error' do
             expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
           end
@@ -253,4 +253,60 @@ describe "inputs/jms" do
       end
     end
   end
+
+  context 'invalid configuration' do
+    let (:jms_config) { super().merge('include_headers' => true, 'include_header' => false) }
+
+    it 'should raise a configuration error' do
+      expect { subject.register }.to raise_error LogStash::ConfigurationError,
+                                                 /Both `include_headers => true` and `include_header => false` options are specified/i
+    end
+  end
+
+  context 'targets', :ecs_compatibility_support do
+
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+      let (:jms_config) do
+        super().merge(
+            'include_body' => false,
+            'include_headers' => true, 'headers_target' => '@metadata',
+            'include_properties' => true, 'properties_target' => '[@metadata]'
+        )
+      end
+
+      let(:jms_timestamp) { 1 }
+
+      let(:jms_message_double) do
+        message = double('jms-message-stub')
+        allow(message).to receive(:jms_timestamp).and_return(jms_timestamp)
+        allow_any_instance_of(described_class).to receive(:map_headers).and_return(
+            'jms_message_id' => 'id1', 'jms_timestamp' => jms_timestamp, 'jms_expiration' => nil
+        )
+        allow(message).to receive(:properties).and_return(:foo => 'bar', 'the-baz' => 42)
+        message
+      end
+
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
+        plugin.register
+        plugin.queue_event(jms_message_double, queue = [])
+        expect(queue.size).to eql 1
+        @event = queue.first
+      end
+
+      it 'sets headers into meta-data' do
+        expect( @event.get('[@metadata][jms_timestamp]') ).to eql jms_timestamp
+        expect( @event.get('[@metadata][jms_message_id]') ).to_not be nil
+      end
+
+      it 'sets properties as meta-data' do
+        expect( @event.get('[@metadata][foo]') ).to eql 'bar'
+        expect( @event.get('[@metadata][the-baz]') ).to eql 42
+      end
+
+    end
+
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jms
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.2.0
 platform: java
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-05 00:00:00.000000000 Z
+date: 2021-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,17 +61,45 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.0.0
-  name: semantic_logger
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-event_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -86,6 +114,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  name: semantic_logger
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -144,8 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Reads events from a Jms Broker