logstash-input-imap 3.0.6 → 3.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfe50e7fb2200f686f84661e1613bc085df5b0eef179b502d0a45c751d9f2747
-  data.tar.gz: 8da74ac7ab6497adee7ea97d710a2fde690770c6add9da6bd4a8d879c3276917
+  metadata.gz: edc2c8598f107e492cecb85f0dc6273b1f4e1147d3042c296131456fa3a1f029
+  data.tar.gz: '09af588e526c930db0a8fe6a5dd86b0965061b81e8ce1b0f49da0912dd06f0a8'
 SHA512:
-  metadata.gz: 83e286cde55af735cc86e63e476ee61674762c4b59942f887cc1478c3e42127cb02a43ebe559cfbf36ce93aa8e9d231d30da5c9cadb37ca242e47fa66c3f635e
-  data.tar.gz: 19d50e2ab991b2bebe610f4a696c927bf6b5f0b9da6817b4ce2946550622d30c5a7edfe236981104f26a2be43b41462da8428b295292207198f48e2fb25e294e
+  metadata.gz: 1e0c068137701bf82698a3dd8546164b0723e97fabd317746ef08fe2594ef1f0c0fe7014be1211689a1401df7ccbbefd06f16015e20f336d7b1fce0182853eae
+  data.tar.gz: de968afeb6d1fb8653f307209c2999c71c32f679792354af4a97d1af71758767748b43b21a561edaf82b27dbcbc28bc084244b1cc577a852261121b304b3156f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.7
+  - Added facility to use IMAP uid to retrieve new mails instead of "NOT SEEN" [#36](https://github.com/logstash-plugins/logstash-input-imap/pull/36)
+
 ## 3.0.6
   - Docs: Set the default_codec doc attribute.
 

data/docs/index.asciidoc

@@ -45,7 +45,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|Yes
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-secure>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-strip_attachments>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-uid_tracking>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-verify_cert>> |<<boolean,boolean>>|No
 |=======================================================================
@@ -56,7 +58,7 @@ input plugins.
 &nbsp;
 
 [id="plugins-{type}s-{plugin}-check_interval"]
-===== `check_interval` 
+===== `check_interval`
 
   * Value type is <<number,number>>
   * Default value is `300`
@@ -64,7 +66,7 @@ input plugins.
 
 
 [id="plugins-{type}s-{plugin}-content_type"]
-===== `content_type` 
+===== `content_type`
 
   * Value type is <<string,string>>
   * Default value is `"text/plain"`
@@ -73,7 +75,7 @@ For multipart messages, use the first part that has this
 content-type as the event message.
 
 [id="plugins-{type}s-{plugin}-delete"]
-===== `delete` 
+===== `delete`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -81,7 +83,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-expunge"]
-===== `expunge` 
+===== `expunge`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -89,7 +91,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-fetch_count"]
-===== `fetch_count` 
+===== `fetch_count`
 
   * Value type is <<number,number>>
   * Default value is `50`
@@ -97,7 +99,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-folder"]
-===== `folder` 
+===== `folder`
 
   * Value type is <<string,string>>
   * Default value is `"INBOX"`
@@ -105,7 +107,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-host"]
-===== `host` 
+===== `host`
 
   * This is a required setting.
   * Value type is <<string,string>>
@@ -114,7 +116,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-lowercase_headers"]
-===== `lowercase_headers` 
+===== `lowercase_headers`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
@@ -122,7 +124,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-password"]
-===== `password` 
+===== `password`
 
   * This is a required setting.
   * Value type is <<password,password>>
@@ -131,7 +133,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-port"]
-===== `port` 
+===== `port`
 
   * Value type is <<number,number>>
   * There is no default value for this setting.
@@ -139,23 +141,52 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-secure"]
-===== `secure` 
+===== `secure`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
 
 
+[id="plugins-{type}s-{plugin}-sincedb_path"]
+===== `sincedb_path`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path of the sincedb database file (keeps track of the UID of the last processed
+mail) that will be written to disk. The default will write sincedb file to
+`<path.data>/plugins/inputs/imap` directory.
+NOTE: it must be a file path and not a directory path.
+
 [id="plugins-{type}s-{plugin}-strip_attachments"]
-===== `strip_attachments` 
+===== `strip_attachments`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
 
 
+[id="plugins-{type}s-{plugin}-uid_tracking"]
+===== `uid_tracking`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+When the IMAP input plugin connects to the mailbox for the first time and
+the UID of the last processed mail is not yet known, the unread mails are
+first downloaded and the UID of the last processed mail is saved. From
+this point on, if `uid_tracking` is set to `true`, all new mail will be
+downloaded regardless of whether they are marked as read or unread. This
+allows users or other services to use the mailbox simultaneously with the
+IMAP input plugin. UID of the last processed mail is always saved regardles
+of the `uid_tracking` value, so you can switch its value as needed. In
+transition from the previous IMAP input plugin version, first process at least
+one mail with `uid_tracking` set to `false` to save the UID of the last
+processed mail and then switch `uid_tracking` to `true`.
+
 [id="plugins-{type}s-{plugin}-user"]
-===== `user` 
+===== `user`
 
   * This is a required setting.
   * Value type is <<string,string>>
@@ -164,7 +195,7 @@ content-type as the event message.
 
 
 [id="plugins-{type}s-{plugin}-verify_cert"]
-===== `verify_cert` 
+===== `verify_cert`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
@@ -176,4 +207,4 @@ content-type as the event message.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/inputs/imap.rb

@@ -29,11 +29,17 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   config :delete, :validate => :boolean, :default => false
   config :expunge, :validate => :boolean, :default => false
   config :strip_attachments, :validate => :boolean, :default => false
-  
+
   # For multipart messages, use the first part that has this
   # content-type as the event message.
   config :content_type, :validate => :string, :default => "text/plain"
 
+  # Whether to use IMAP uid to track last processed message
+  config :uid_tracking, :validate => :boolean, :default => false
+
+  # Path to file with last run time metadata
+  config :sincedb_path, :validate => :string, :required => false
+
   def register
     require "net/imap" # in stdlib
     require "mail" # gem 'mail'
@@ -50,6 +56,22 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
       end
     end
 
+    # Load last processed IMAP uid from file if exists
+    if @sincedb_path.nil?
+      datapath = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "imap")
+      # Ensure that the filepath exists before writing, since it's deeply nested.
+      FileUtils::mkdir_p datapath
+      @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
+    end
+    if File.directory?(@sincedb_path)
+      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
+    end
+    @logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
+    if File.exist?(@sincedb_path)
+      @uid_last_value = File.read(@sincedb_path).to_i
+      @logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
+    end
+
     @content_type_re = Regexp.new("^" + @content_type)
   end # def register
 
@@ -75,34 +97,56 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
     # EOFError, OpenSSL::SSL::SSLError
     imap = connect
     imap.select(@folder)
-    ids = imap.search("NOT SEEN")
+    if @uid_tracking && @uid_last_value
+      # If there are no new messages, uid_search returns @uid_last_value
+      # because it is the last message, so we need to delete it.
+      ids = imap.uid_search(["UID", (@uid_last_value+1..-1)]).delete_if { |uid|
+        uid <= @uid_last_value
+      }
+    else
+      ids = imap.uid_search("NOT SEEN")
+    end
 
     ids.each_slice(@fetch_count) do |id_set|
-      items = imap.fetch(id_set, "RFC822")
+      items = imap.uid_fetch(id_set, ["BODY.PEEK[]", "UID"])
       items.each do |item|
-        next unless item.attr.has_key?("RFC822")
-        mail = Mail.read_from_string(item.attr["RFC822"])
+        next unless item.attr.has_key?("BODY[]")
+        mail = Mail.read_from_string(item.attr["BODY[]"])
         if @strip_attachments
           queue << parse_mail(mail.without_attachments!)
         else
           queue << parse_mail(mail)
         end
+        # Mark message as processed
+        @uid_last_value = item.attr["UID"]
+        imap.uid_store(@uid_last_value, '+FLAGS', @delete || @expunge ? :Deleted : :Seen)
+
+        # Stop message processing if it is requested
+        break if stop?
       end
 
-      imap.store(id_set, '+FLAGS', @delete ? :Deleted : :Seen)
-  
-    end
+      # Expunge deleted messages
+      imap.expunge() if @expunge
 
-  # Enable an 'expunge' IMAP command after the items.each loop
-    if @expunge 
-    # Force messages to be marked as "Deleted", the above may or may not be working as expected. "Seen" means nothing if you are going to
-    # delete a message after processing.
-      imap.store(id_set, '+FLAGS', [:Deleted])
-      imap.expunge()
+      # Stop message fetching if it is requested
+      break if stop?
     end
 
-    imap.close
-    imap.disconnect
+  rescue => e
+    @logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace)
+    # Do not raise error, check_mail will be invoked in the next run time
+
+  ensure
+    # Close the connection (and ignore errors)
+    imap.close rescue nil
+    imap.disconnect rescue nil
+
+    # Always save @uid_last_value so when tracking is switched from
+    # "NOT SEEN" to "UID" we will continue from first unprocessed message
+    if @uid_last_value
+      @logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
+      File.write(@sincedb_path, @uid_last_value)
+    end
   end
 
   def parse_mail(mail)

data/logstash-input-imap.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-imap'
-  s.version         = '3.0.6'
+  s.version         = '3.0.7'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads mail from an IMAP server"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/imap_spec.rb

@@ -25,7 +25,7 @@ describe LogStash::Inputs::IMAP do
         allow(imap).to receive(:store)
         allow(ids).to receive(:each_slice).and_return([])
 
-        allow(imap).to receive(:search).with("NOT SEEN").and_return(ids)
+        allow(imap).to receive(:uid_search).with("NOT SEEN").and_return(ids)
         allow(Net::IMAP).to receive(:new).and_return(imap)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-imap
 version: !ruby/object:Gem::Version
-  version: 3.0.6
+  version: 3.0.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2019-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads mail from an IMAP server