logstash-input-imap 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3660e0f1f79393223c4350026f3702f6c755b374bd1fcd8d9ca64fe60be0b17d
-  data.tar.gz: c65caa46e847c7471f67272e569a122d6939a11274a743d7c587afe177503c66
+  metadata.gz: 61f403ebf9de18a49ba8f7f930ea04484670701909dd70f8d5e67a31706c0806
+  data.tar.gz: 1ec42dda0d258f515c8e2cb67e0ef3f3bd5e7fe61dc16b5916bb3afa020d1dab
 SHA512:
-  metadata.gz: 21f6ab147a10aba0648f9257f22a6d7eb0d78ec7026529aed2196cfc0c0c5560ab7b220c0f3ab01e96203752a5aaf8ca72b9af0363a478797f4c198de5c4497d
-  data.tar.gz: 8396ea0d0196edec2b346bfb6a0a375c55961465f4dac5f4738f243058024a0fe179b02caf6bbf8b5c71ab8e9c7e2bd6371a25b7c1c69f0fe3b7a8e99303660b
+  metadata.gz: e588b5e14d17cc88139fd5a869102a47e42f17f9d0777f59808c2f599bbdf0d76ff4566f21a8ebe4e8761a0665ec10b68c69071beda4be8e40ce541c9de245b3
+  data.tar.gz: baf7c0d8f3ee32cd367c2537b9ede816a1aa2d39741edbf80ffec53792bd80b4395864afc4cd4645e796069f2a79a73dac44551bd6e35553cec4685ee6b7f368

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.2.0
+  - Feat: ECS compatibility [#55](https://github.com/logstash-plugins/logstash-input-imap/pull/55)
+    * added (optional) `headers_target` configuration option
+    * added (optional) `attachments_target` configuration option
+  - Fix: plugin should not close `$stdin`, while being stopped 
+
 ## 3.1.0
   - Adds an option to recursively search the message parts for attachment and inline attachment filenames. If the save_attachments option is set to true, the content of attachments is included in the `attachments.data` field. The attachment data can then be used by the Elasticsearch Ingest Attachment Processor Plugin.
   [#48](https://github.com/logstash-plugins/logstash-input-imap/pull/48)

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-imap.svg)](https://travis-ci.org/logstash-plugins/logstash-input-imap)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-imap.svg)](https://travis-ci.com/logstash-plugins/logstash-input-imap)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -26,6 +26,15 @@ Read mails from IMAP server
 Periodically scan an IMAP folder (`INBOX` by default) and move any read messages
 to the trash.
 
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+The plugin includes sensible defaults that change based on <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>.
+When ECS compatibility is disabled, mail headers and attachments are targeted at the root level.
+When targeting an ECS version, headers and attachments target `@metadata` sub-fields unless configured otherwise in order
+to avoid conflict with ECS fields.
+See <<plugins-{type}s-{plugin}-headers_target>>, and <<plugins-{type}s-{plugin}-attachments_target>>.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Imap Input Configuration Options
 
@@ -34,12 +43,15 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-attachments_target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-check_interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-content_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-expunge>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fetch_count>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-folder>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-headers_target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-lowercase_headers>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|Yes
@@ -58,6 +70,16 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-attachments_target"]
+===== `attachments_target`
+
+  * Value type is <<string,string>>
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: `"[attachments]"`
+    ** ECS Compatibility enabled: `"[@metadata][input][imap][attachments]"
+
+The name of the field under which mail attachments information will be added, if <<plugins-{type}s-{plugin}-save_attachments>> is set.
+
 [id="plugins-{type}s-{plugin}-check_interval"]
 ===== `check_interval`
 
@@ -72,8 +94,7 @@ input plugins.
   * Value type is <<string,string>>
   * Default value is `"text/plain"`
 
-For multipart messages, use the first part that has this
-content-type as the event message.
+For multipart messages, use the first part that has this content-type as the event message.
 
 [id="plugins-{type}s-{plugin}-delete"]
 ===== `delete`
@@ -83,6 +104,21 @@ content-type as the event message.
 
 
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: does not use ECS-compatible field names (for example, `From` header field is added to the event)
+    ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `From` header is added as metadata)
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+The value of this setting affects the _default_ value of <<plugins-{type}s-{plugin}-headers_target>> and
+<<plugins-{type}s-{plugin}-attachments_target>>.
+
 [id="plugins-{type}s-{plugin}-expunge"]
 ===== `expunge`
 
@@ -107,6 +143,19 @@ content-type as the event message.
 
 
 
+[id="plugins-{type}s-{plugin}-headers_target"]
+===== `headers_target`
+
+  * Value type is <<string,string>>
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: no default value (for example, the subject header is stored under the `"subject"` name)
+    ** ECS Compatibility enabled: `"[@metadata][input][imap][headers]"`
+
+The name of the field under which mail headers will be added.
+
+Setting `headers_target => ''` skips headers processing and no header is added to the event.
+Except the date header, if present, which is always used as the event's `@timestamp`.
+
 [id="plugins-{type}s-{plugin}-host"]
 ===== `host`
 

data/lib/logstash/inputs/imap.rb

@@ -3,13 +3,22 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/timestamp"
 require "stud/interval"
-require "socket" # for Socket.gethostname
+require 'fileutils'
+
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 
 # Read mails from IMAP server
 #
 # Periodically scan an IMAP folder (`INBOX` by default) and move any read messages
 # to the trash.
 class LogStash::Inputs::IMAP < LogStash::Inputs::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "imap"
 
   default :codec, "plain"
@@ -24,15 +33,23 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
 
   config :folder, :validate => :string, :default => 'INBOX'
   config :fetch_count, :validate => :number, :default => 50
-  config :lowercase_headers, :validate => :boolean, :default => true
   config :check_interval, :validate => :number, :default => 300
+
+  config :lowercase_headers, :validate => :boolean, :default => true
+
+  config :headers_target, :validate => :field_reference # ECS default: [@metadata][input][imap][headers]
+
   config :delete, :validate => :boolean, :default => false
   config :expunge, :validate => :boolean, :default => false
+
   config :strip_attachments, :validate => :boolean, :default => false
   config :save_attachments, :validate => :boolean, :default => false
 
-  # For multipart messages, use the first part that has this
-  # content-type as the event message.
+  # Legacy default: [attachments]
+  # ECS default: [@metadata][input][imap][attachments]
+  config :attachments_target, :validate => :field_reference
+
+  # For multipart messages, use the first part that has this content-type as the event message.
   config :content_type, :validate => :string, :default => "text/plain"
 
   # Whether to use IMAP uid to track last processed message
@@ -41,6 +58,32 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   # Path to file with last run time metadata
   config :sincedb_path, :validate => :string, :required => false
 
+  def initialize(*params)
+    super
+
+    if original_params.include?('headers_target')
+      @headers_target = normalize_field_ref(headers_target)
+    else
+      # NOTE: user specified `headers_target => ''` means disable headers (@headers_target == nil)
+      # unlike our default here (@headers_target == '') causes setting headers at top level ...
+      @headers_target = ecs_compatibility != :disabled ? '[@metadata][input][imap][headers]' : ''
+    end
+
+    if original_params.include?('attachments_target')
+      @attachments_target = normalize_field_ref(attachments_target)
+    else
+      @attachments_target = ecs_compatibility != :disabled ? '[@metadata][input][imap][attachments]' : '[attachments]'
+    end
+  end
+
+  # @note a '' target value is normalized to nil
+  def normalize_field_ref(target)
+    return nil if target.nil? || target.empty?
+    # so we can later event.set("#{target}[#{name}]", ...)
+    target.match?(/\A[^\[\]]+\z/) ? "[#{target}]" : target
+  end
+  private :normalize_field_ref
+
   def register
     require "net/imap" # in stdlib
     require "mail" # gem 'mail'
@@ -63,14 +106,15 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
       # Ensure that the filepath exists before writing, since it's deeply nested.
       FileUtils::mkdir_p datapath
       @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
+      @logger.debug? && @logger.debug("Generated sincedb path", sincedb_path: @sincedb_path)
     end
-    if File.directory?(@sincedb_path)
-      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
-    end
-    @logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
+    @logger.info("Using", sincedb_path: @sincedb_path)
     if File.exist?(@sincedb_path)
+      if File.directory?(@sincedb_path)
+        raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
+      end
       @uid_last_value = File.read(@sincedb_path).to_i
-      @logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
+      @logger.debug? && @logger.debug("Loaded from sincedb", uid_last_value: @uid_last_value)
     end
 
     @content_type_re = Regexp.new("^" + @content_type)
@@ -136,7 +180,6 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   rescue => e
     @logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace)
     # Do not raise error, check_mail will be invoked in the next run time
-
   ensure
     # Close the connection (and ignore errors)
     imap.close rescue nil
@@ -145,7 +188,7 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
     # Always save @uid_last_value so when tracking is switched from
     # "NOT SEEN" to "UID" we will continue from first unprocessed message
     if @uid_last_value
-      @logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
+      @logger.debug? && @logger.debug("Saving to sincedb", uid_last_value: @uid_last_value)
       File.write(@sincedb_path, @uid_last_value)
     end
   end
@@ -164,7 +207,8 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
 
   def parse_mail(mail)
     # Add a debug message so we can track what message might cause an error later
-    @logger.debug? && @logger.debug("Working with message_id", :message_id => mail.message_id)
+    @logger.debug? && @logger.debug("Processing mail", message_id: mail.message_id)
+
     # TODO(sissel): What should a multipart message look like as an event?
     # For now, just take the plain-text part and set it as the message.
     if mail.parts.count == 0
@@ -183,35 +227,11 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
       # Use the 'Date' field as the timestamp
       event.timestamp = LogStash::Timestamp.new(mail.date.to_time)
 
-      # Add fields: Add message.header_fields { |h| h.name=> h.value }
-      mail.header_fields.each do |header|
-        # 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in String form
-        name = @lowercase_headers ? header.name.to_s.downcase : header.name.to_s
-        # Call .decoded on the header in case it's in encoded-word form.
-        # Details at:
-        #   https://github.com/mikel/mail/blob/master/README.md#encodings
-        #   http://tools.ietf.org/html/rfc2047#section-2
-        value = transcode_to_utf8(header.decoded.to_s)
-
-        # Assume we already processed the 'date' above.
-        next if name == "Date"
-
-        case (field = event.get(name))
-        when String
-          # promote string to array if a header appears multiple times
-          # (like 'received')
-          event.set(name, [field, value])
-        when Array
-          field << value
-          event.set(name, field)
-        when nil
-          event.set(name, value)
-        end
-      end
+      process_headers(mail, event) if @headers_target
 
       # Add attachments
-      if attachments && attachments.length > 0
-        event.set('attachments', attachments)
+      if attachments && attachments.length > 0 && @attachments_target
+        event.set(@attachments_target, attachments)
       end
 
       decorate(event)
@@ -219,9 +239,35 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
     end
   end
 
+  def process_headers(mail, event)
+    # Add fields: Add message.header_fields { |h| h.name=> h.value }
+    mail.header_fields.each do |header|
+      # 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in String form
+      name = header.name.to_s
+      name = name.downcase if @lowercase_headers
+
+      # Call .decoded on the header in case it's in encoded-word form.
+      # Details at:
+      #   https://github.com/mikel/mail/blob/master/README.md#encodings
+      #   http://tools.ietf.org/html/rfc2047#section-2
+      value = transcode_to_utf8(header.decoded)
+
+      targeted_name = "#{@headers_target}[#{name}]"
+      case (field = event.get(targeted_name))
+      when String
+        # promote string to array if a header appears multiple times (like 'received')
+        event.set(targeted_name, [field, value])
+      when Array
+        field << value
+        event.set(targeted_name, field)
+      when nil
+        event.set(targeted_name, value)
+      end
+    end
+  end
+
   def stop
     Stud.stop!(@run_thread)
-    $stdin.close
   end
 
   private
@@ -230,8 +276,7 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   # the mail gem will set the correct encoding on header strings decoding
   # and we want to transcode it to utf8
   def transcode_to_utf8(s)
-    unless s.nil?
-      s.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
-    end
+    return nil if s.nil?
+    s.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
   end
 end

data/logstash-input-imap.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-imap'
-  s.version         = '3.1.0'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads mail from an IMAP server"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,11 +21,12 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'mail', '~> 2.6.3'
   s.add_runtime_dependency 'mime-types', '2.6.2'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
 
   s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'insist'
 end

data/spec/inputs/imap_spec.rb

@@ -1,13 +1,12 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
-require "insist"
 require "logstash/devutils/rspec/shared_examples"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 require "logstash/inputs/imap"
 require "mail"
 require "net/imap"
 require "base64"
 
-
 describe LogStash::Inputs::IMAP do
 
   context "when interrupting the plugin" do
@@ -36,7 +35,7 @@ describe LogStash::Inputs::IMAP do
 
 end
 
-describe LogStash::Inputs::IMAP do
+describe LogStash::Inputs::IMAP, :ecs_compatibility_support do
   user = "logstash"
   password = "secret"
   msg_time = Time.new
@@ -45,129 +44,224 @@ describe LogStash::Inputs::IMAP do
   msg_binary = "\x42\x43\x44"
   msg_unencoded = "raw text 🐐"
 
-  subject do
+  let(:config) do
+    { "host" => "localhost", "user" => "#{user}", "password" => "#{password}" }
+  end
+
+  subject(:input) do
+    LogStash::Inputs::IMAP.new config
+  end
+
+  let(:mail) do
     Mail.new do
       from     "me@example.com"
       to       "you@example.com"
       subject  "logstash imap input test"
       date     msg_time
       body     msg_text
+      message_id '<123@message.id>' # 'Message-ID' header
+      # let's have some headers:
+      header['X-Priority'] = '3'
+      header['X-Bot-ID'] = '111'
+      header['X-AES-Category'] = 'LEGIT'
+      header['X-Spam-Category'] = 'LEGIT'
+      header['Spam-Stopper-Id'] = '464bbb1a-1b86-4006-8a09-ce797fb56346'
+      header['Spam-Stopper-v2'] = 'Yes'
+      header['X-Mailer'] = 'Microsoft Outlook Express 6.00.2800.1106'
+      header['X-MimeOLE'] = 'Produced By Microsoft MimeOLE V6.00.2800.1106'
       add_file :filename => "some.html", :content => msg_html
       add_file :filename => "image.png", :content => msg_binary
       add_file :filename => "unencoded.data", :content => msg_unencoded, :content_transfer_encoding => "7bit"
     end
   end
 
-  context "with both text and html parts" do
+  before do
+    input.register
+  end
+
+  ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+    let(:ecs_compatibility?) { ecs_select.active_mode != :disabled }
+
+    let (:config) { super().merge('ecs_compatibility' => ecs_select.active_mode) }
+
     context "when no content-type selected" do
       it "should select text/plain part" do
-        config = {"type" => "imap", "host" => "localhost",
-                  "user" => "#{user}", "password" => "#{password}"}
-
-        input = LogStash::Inputs::IMAP.new config
-        input.register
-        event = input.parse_mail(subject)
-        insist { event.get("message") } == msg_text
+        event = input.parse_mail(mail)
+        expect( event.get("message") ).to eql msg_text
       end
     end
 
     context "when text/html content-type selected" do
+      let(:config) { super().merge("content_type" => "text/html") }
+
       it "should select text/html part" do
-        config = {"type" => "imap", "host" => "localhost",
-                  "user" => "#{user}", "password" => "#{password}",
-                  "content_type" => "text/html"}
+        event = input.parse_mail(mail)
+        expect( event.get("message") ).to eql msg_html
+      end
+    end
+
+    context "mail headers" do
+      let(:config) { super().merge("lowercase_headers" => true) } # default
 
-        input = LogStash::Inputs::IMAP.new config
-        input.register
-        event = input.parse_mail(subject)
-        insist { event.get("message") } == msg_html
+      before { @event = input.parse_mail(mail) }
+
+      it "sets all header fields" do
+        if ecs_compatibility?
+          expect( @event.get("[@metadata][input][imap][headers][x-spam-category]") ).to eql 'LEGIT'
+          expect( @event.get("[@metadata][input][imap][headers][x-aes-category]") ).to eql 'LEGIT'
+          expect( @event.get("[@metadata][input][imap][headers][x-bot-id]") ).to eql '111'
+          ['spam-stopper-id', 'spam-stopper-v2', 'x-mimeole', 'message-id', 'x-priority'].each do |name|
+            expect( @event.include?("[@metadata][input][imap][headers][#{name}]") ).to be true
+          end
+          expect( @event.get("[@metadata][input][imap][headers][from]") ).to eql 'me@example.com'
+          expect( @event.get("[@metadata][input][imap][headers][to]") ).to eql 'you@example.com'
+          expect( @event.get("[@metadata][input][imap][headers][subject]") ).to eql 'logstash imap input test'
+        else
+          expect( @event.get("x-spam-category") ).to eql 'LEGIT'
+          expect( @event.get("x-aes-category") ).to eql 'LEGIT'
+          expect( @event.get("x-bot-id") ).to eql '111'
+          ['spam-stopper-id', 'spam-stopper-v2', 'x-mimeole', 'message-id', 'x-priority'].each do |name|
+            expect( @event.include?(name) ).to be true
+          end
+          expect( @event.get("from") ).to eql 'me@example.com'
+          expect( @event.get("to") ).to eql 'you@example.com'
+          expect( @event.get("subject") ).to eql 'logstash imap input test'
+        end
+      end
+
+      it 'does include the date header' do
+        expect( @event.include?('date') ).to be true unless ecs_compatibility?
+        expect( @event.include?('Date') ).to be false
       end
     end
-  end
 
-  context "when subject is in RFC 2047 encoded-word format" do
-    it "should be decoded" do
-      subject.subject = "=?iso-8859-1?Q?foo_:_bar?="
-      config = {"type" => "imap", "host" => "localhost",
-                "user" => "#{user}", "password" => "#{password}"}
+    context "mail headers (not lower-cased)" do
+      let(:config) { super().merge("lowercase_headers" => false) }
+
+      before { @event = input.parse_mail(mail) }
 
-      input = LogStash::Inputs::IMAP.new config
-      input.register
-      event = input.parse_mail(subject)
-      insist { event.get("subject") } == "foo : bar"
+      it "sets all header fields" do
+        if ecs_compatibility?
+          expect( @event.get("[@metadata][input][imap][headers][X-Spam-Category]") ).to eql 'LEGIT'
+          expect( @event.get("[@metadata][input][imap][headers][X-AES-Category]") ).to eql 'LEGIT'
+          expect( @event.get("[@metadata][input][imap][headers][X-Bot-ID]") ).to eql '111'
+          ['Spam-Stopper-Id', 'Spam-Stopper-v2', 'X-MimeOLE', 'Message-ID', 'X-Priority'].each do |name|
+            expect( @event.include?("[@metadata][input][imap][headers][#{name}]") ).to be true
+          end
+          expect( @event.get("[@metadata][input][imap][headers][From]") ).to eql 'me@example.com'
+          expect( @event.get("[@metadata][input][imap][headers][To]") ).to eql 'you@example.com'
+          expect( @event.get("[@metadata][input][imap][headers][Subject]") ).to eql 'logstash imap input test'
+        else
+          expect( @event.get("X-Spam-Category") ).to eql 'LEGIT'
+          expect( @event.get("X-AES-Category") ).to eql 'LEGIT'
+          expect( @event.get("X-Bot-ID") ).to eql '111'
+          ['Spam-Stopper-Id', 'Spam-Stopper-v2', 'X-MimeOLE', 'Message-ID', 'X-Priority'].each do |name|
+            expect( @event.include?(name) ).to be true
+          end
+          expect( @event.get("From") ).to eql 'me@example.com'
+          expect( @event.get("To") ).to eql 'you@example.com'
+          expect( @event.get("Subject") ).to eql 'logstash imap input test'
+        end
+      end
+
+      it 'does include the date header' do
+        expect( @event.include?('Date') ).to be true unless ecs_compatibility?
+      end
     end
-  end
 
-  context "with multiple values for same header" do
-    it "should add 2 values as array in event" do
-      subject.received = "test1"
-      subject.received = "test2"
+    context "headers_target => ''" do
+      let(:config) { super().merge("headers_target" => '') }
+
+      before { @event = input.parse_mail(mail) }
 
-      config = {"type" => "imap", "host" => "localhost",
-                "user" => "#{user}", "password" => "#{password}"}
+      it "does not set any header fields" do
+        ['From', 'To', 'Subject', 'subject', 'Date', 'date'].each do |name|
+          expect( @event.include?(name) ).to be false # legacy
+          expect( @event.include?("[@metadata][input][imap][headers][#{name}]") ).to be false # ecs
+        end
+      end
+    end
 
-      input = LogStash::Inputs::IMAP.new config
-      input.register
-      event = input.parse_mail(subject)
-      insist { event.get("received") } == ["test1", "test2"]
+    context "when subject is in RFC 2047 encoded-word format" do
+      before do
+        mail.subject = "=?iso-8859-1?Q?foo_:_bar?="
+      end
+
+      it "should be decoded" do
+        event = input.parse_mail(mail)
+        if ecs_compatibility?
+          expect( event.get("[@metadata][input][imap][headers][subject]") ).to eql "foo : bar"
+        else
+          expect( event.get("subject") ).to eql "foo : bar"
+        end
+      end
     end
 
-    it "should add more than 2 values as array in event" do
-      subject.received = "test1"
-      subject.received = "test2"
-      subject.received = "test3"
+    context "with multiple values for same header" do
+      it "should add 2 values as array in event" do
+        mail.received = "test1"
+        mail.received = "test2"
 
-      config = {"type" => "imap", "host" => "localhost",
-                "user" => "#{user}", "password" => "#{password}"}
+        event = input.parse_mail(mail)
+        expected_value = ["test1", "test2"]
+        if ecs_compatibility?
+          expect( event.get("[@metadata][input][imap][headers][received]") ).to eql expected_value
+        else
+          expect( event.get("received") ).to eql expected_value
+        end
+      end
+
+      it "should add more than 2 values as array in event" do
+        mail.received = "test1"
+        mail.received = "test2"
+        mail.received = "test3"
 
-      input = LogStash::Inputs::IMAP.new config
-      input.register
-      event = input.parse_mail(subject)
-      insist { event.get("received") } == ["test1", "test2", "test3"]
+        event = input.parse_mail(mail)
+        expected_value = ["test1", "test2", "test3"]
+        if ecs_compatibility?
+          expect( event.get("[@metadata][input][imap][headers][received]") ).to eql expected_value
+        else
+          expect( event.get("received") ).to eql expected_value
+        end
+      end
     end
-  end
 
-  context "when a header field is nil" do
-    it "should parse mail" do
-      subject.header['X-Custom-Header'] = nil
-      config = {"type" => "imap", "host" => "localhost",
-                "user" => "#{user}", "password" => "#{password}"}
+    context "when a header field is nil" do
+      it "should parse mail" do
+        mail.header['X-Custom-Header'] = nil
 
-      input = LogStash::Inputs::IMAP.new config
-      input.register
-      event = input.parse_mail(subject)
-      insist { event.get("message") } == msg_text
+        event = input.parse_mail(mail)
+        expect( event.get("message") ).to eql msg_text
+      end
     end
-  end
 
-  context "with attachments" do
-    it "should extract filenames" do
-      config = {"type" => "imap", "host" => "localhost",
-                "user" => "#{user}", "password" => "#{password}"}
-
-      input = LogStash::Inputs::IMAP.new config
-      input.register
-      event = input.parse_mail(subject)
-      insist { event.get("attachments") } == [
-        {"filename"=>"some.html"},
-        {"filename"=>"image.png"},
-        {"filename"=>"unencoded.data"}
-      ]
+    context "attachments" do
+      it "should extract filenames" do
+        event = input.parse_mail(mail)
+        target = ecs_compatibility? ? '[@metadata][input][imap][attachments]' : 'attachments'
+        expect( event.get(target) ).to eql [
+                                                {"filename"=>"some.html"},
+                                                {"filename"=>"image.png"},
+                                                {"filename"=>"unencoded.data"}
+                                            ]
+      end
     end
 
-    it "should extract the encoded content" do
-      config = {"type" => "imap", "host" => "localhost",
-        "user" => "#{user}", "password" => "#{password}",
-        "save_attachments" => true}
+    context "with attachments saving" do
+      let(:config) { super().merge("save_attachments" => true) }
 
-      input = LogStash::Inputs::IMAP.new config
-      input.register
-      event = input.parse_mail(subject)
-      insist { event.get("attachments") } == [
-        {"data"=> Base64.encode64(msg_html).encode(crlf_newline: true), "filename"=>"some.html"},
-        {"data"=> Base64.encode64(msg_binary).encode(crlf_newline: true), "filename"=>"image.png"},
-        {"data"=> msg_unencoded, "filename"=>"unencoded.data"}
-      ]
+      it "should extract the encoded content" do
+        event = input.parse_mail(mail)
+        target = ecs_compatibility? ? '[@metadata][input][imap][attachments]' : 'attachments'
+        expect( event.get(target) ).to eql [
+                                                {"data"=> Base64.encode64(msg_html).encode(crlf_newline: true), "filename"=>"some.html"},
+                                                {"data"=> Base64.encode64(msg_binary).encode(crlf_newline: true), "filename"=>"image.png"},
+                                                {"data"=> msg_unencoded, "filename"=>"unencoded.data"}
+                                            ]
       end
+    end
+
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-imap
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-27 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,34 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -100,20 +128,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: insist
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -153,8 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Reads mail from an IMAP server