logstash-input-imap 3.0.6 → 3.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dfe50e7fb2200f686f84661e1613bc085df5b0eef179b502d0a45c751d9f2747
4
- data.tar.gz: 8da74ac7ab6497adee7ea97d710a2fde690770c6add9da6bd4a8d879c3276917
3
+ metadata.gz: edc2c8598f107e492cecb85f0dc6273b1f4e1147d3042c296131456fa3a1f029
4
+ data.tar.gz: '09af588e526c930db0a8fe6a5dd86b0965061b81e8ce1b0f49da0912dd06f0a8'
5
5
  SHA512:
6
- metadata.gz: 83e286cde55af735cc86e63e476ee61674762c4b59942f887cc1478c3e42127cb02a43ebe559cfbf36ce93aa8e9d231d30da5c9cadb37ca242e47fa66c3f635e
7
- data.tar.gz: 19d50e2ab991b2bebe610f4a696c927bf6b5f0b9da6817b4ce2946550622d30c5a7edfe236981104f26a2be43b41462da8428b295292207198f48e2fb25e294e
6
+ metadata.gz: 1e0c068137701bf82698a3dd8546164b0723e97fabd317746ef08fe2594ef1f0c0fe7014be1211689a1401df7ccbbefd06f16015e20f336d7b1fce0182853eae
7
+ data.tar.gz: de968afeb6d1fb8653f307209c2999c71c32f679792354af4a97d1af71758767748b43b21a561edaf82b27dbcbc28bc084244b1cc577a852261121b304b3156f
@@ -1,3 +1,6 @@
1
+ ## 3.0.7
2
+ - Added facility to use IMAP uid to retrieve new mails instead of "NOT SEEN" [#36](https://github.com/logstash-plugins/logstash-input-imap/pull/36)
3
+
1
4
  ## 3.0.6
2
5
  - Docs: Set the default_codec doc attribute.
3
6
 
@@ -45,7 +45,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
45
45
  | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|Yes
46
46
  | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
47
47
  | <<plugins-{type}s-{plugin}-secure>> |<<boolean,boolean>>|No
48
+ | <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
48
49
  | <<plugins-{type}s-{plugin}-strip_attachments>> |<<boolean,boolean>>|No
50
+ | <<plugins-{type}s-{plugin}-uid_tracking>> |<<boolean,boolean>>|No
49
51
  | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|Yes
50
52
  | <<plugins-{type}s-{plugin}-verify_cert>> |<<boolean,boolean>>|No
51
53
  |=======================================================================
@@ -56,7 +58,7 @@ input plugins.
56
58
  &nbsp;
57
59
 
58
60
  [id="plugins-{type}s-{plugin}-check_interval"]
59
- ===== `check_interval`
61
+ ===== `check_interval`
60
62
 
61
63
  * Value type is <<number,number>>
62
64
  * Default value is `300`
@@ -64,7 +66,7 @@ input plugins.
64
66
 
65
67
 
66
68
  [id="plugins-{type}s-{plugin}-content_type"]
67
- ===== `content_type`
69
+ ===== `content_type`
68
70
 
69
71
  * Value type is <<string,string>>
70
72
  * Default value is `"text/plain"`
@@ -73,7 +75,7 @@ For multipart messages, use the first part that has this
73
75
  content-type as the event message.
74
76
 
75
77
  [id="plugins-{type}s-{plugin}-delete"]
76
- ===== `delete`
78
+ ===== `delete`
77
79
 
78
80
  * Value type is <<boolean,boolean>>
79
81
  * Default value is `false`
@@ -81,7 +83,7 @@ content-type as the event message.
81
83
 
82
84
 
83
85
  [id="plugins-{type}s-{plugin}-expunge"]
84
- ===== `expunge`
86
+ ===== `expunge`
85
87
 
86
88
  * Value type is <<boolean,boolean>>
87
89
  * Default value is `false`
@@ -89,7 +91,7 @@ content-type as the event message.
89
91
 
90
92
 
91
93
  [id="plugins-{type}s-{plugin}-fetch_count"]
92
- ===== `fetch_count`
94
+ ===== `fetch_count`
93
95
 
94
96
  * Value type is <<number,number>>
95
97
  * Default value is `50`
@@ -97,7 +99,7 @@ content-type as the event message.
97
99
 
98
100
 
99
101
  [id="plugins-{type}s-{plugin}-folder"]
100
- ===== `folder`
102
+ ===== `folder`
101
103
 
102
104
  * Value type is <<string,string>>
103
105
  * Default value is `"INBOX"`
@@ -105,7 +107,7 @@ content-type as the event message.
105
107
 
106
108
 
107
109
  [id="plugins-{type}s-{plugin}-host"]
108
- ===== `host`
110
+ ===== `host`
109
111
 
110
112
  * This is a required setting.
111
113
  * Value type is <<string,string>>
@@ -114,7 +116,7 @@ content-type as the event message.
114
116
 
115
117
 
116
118
  [id="plugins-{type}s-{plugin}-lowercase_headers"]
117
- ===== `lowercase_headers`
119
+ ===== `lowercase_headers`
118
120
 
119
121
  * Value type is <<boolean,boolean>>
120
122
  * Default value is `true`
@@ -122,7 +124,7 @@ content-type as the event message.
122
124
 
123
125
 
124
126
  [id="plugins-{type}s-{plugin}-password"]
125
- ===== `password`
127
+ ===== `password`
126
128
 
127
129
  * This is a required setting.
128
130
  * Value type is <<password,password>>
@@ -131,7 +133,7 @@ content-type as the event message.
131
133
 
132
134
 
133
135
  [id="plugins-{type}s-{plugin}-port"]
134
- ===== `port`
136
+ ===== `port`
135
137
 
136
138
  * Value type is <<number,number>>
137
139
  * There is no default value for this setting.
@@ -139,23 +141,52 @@ content-type as the event message.
139
141
 
140
142
 
141
143
  [id="plugins-{type}s-{plugin}-secure"]
142
- ===== `secure`
144
+ ===== `secure`
143
145
 
144
146
  * Value type is <<boolean,boolean>>
145
147
  * Default value is `true`
146
148
 
147
149
 
148
150
 
151
+ [id="plugins-{type}s-{plugin}-sincedb_path"]
152
+ ===== `sincedb_path`
153
+
154
+ * Value type is <<string,string>>
155
+ * There is no default value for this setting.
156
+
157
+ Path of the sincedb database file (keeps track of the UID of the last processed
158
+ mail) that will be written to disk. The default will write sincedb file to
159
+ `<path.data>/plugins/inputs/imap` directory.
160
+ NOTE: it must be a file path and not a directory path.
161
+
149
162
  [id="plugins-{type}s-{plugin}-strip_attachments"]
150
- ===== `strip_attachments`
163
+ ===== `strip_attachments`
151
164
 
152
165
  * Value type is <<boolean,boolean>>
153
166
  * Default value is `false`
154
167
 
155
168
 
156
169
 
170
+ [id="plugins-{type}s-{plugin}-uid_tracking"]
171
+ ===== `uid_tracking`
172
+
173
+ * Value type is <<boolean,boolean>>
174
+ * Default value is `false`
175
+
176
+ When the IMAP input plugin connects to the mailbox for the first time and
177
+ the UID of the last processed mail is not yet known, the unread mails are
178
+ first downloaded and the UID of the last processed mail is saved. From
179
+ this point on, if `uid_tracking` is set to `true`, all new mail will be
180
+ downloaded regardless of whether they are marked as read or unread. This
181
+ allows users or other services to use the mailbox simultaneously with the
182
+ IMAP input plugin. UID of the last processed mail is always saved regardles
183
+ of the `uid_tracking` value, so you can switch its value as needed. In
184
+ transition from the previous IMAP input plugin version, first process at least
185
+ one mail with `uid_tracking` set to `false` to save the UID of the last
186
+ processed mail and then switch `uid_tracking` to `true`.
187
+
157
188
  [id="plugins-{type}s-{plugin}-user"]
158
- ===== `user`
189
+ ===== `user`
159
190
 
160
191
  * This is a required setting.
161
192
  * Value type is <<string,string>>
@@ -164,7 +195,7 @@ content-type as the event message.
164
195
 
165
196
 
166
197
  [id="plugins-{type}s-{plugin}-verify_cert"]
167
- ===== `verify_cert`
198
+ ===== `verify_cert`
168
199
 
169
200
  * Value type is <<boolean,boolean>>
170
201
  * Default value is `true`
@@ -176,4 +207,4 @@ content-type as the event message.
176
207
  [id="plugins-{type}s-{plugin}-common-options"]
177
208
  include::{include_path}/{type}.asciidoc[]
178
209
 
179
- :default_codec!:
210
+ :default_codec!:
@@ -29,11 +29,17 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
29
29
  config :delete, :validate => :boolean, :default => false
30
30
  config :expunge, :validate => :boolean, :default => false
31
31
  config :strip_attachments, :validate => :boolean, :default => false
32
-
32
+
33
33
  # For multipart messages, use the first part that has this
34
34
  # content-type as the event message.
35
35
  config :content_type, :validate => :string, :default => "text/plain"
36
36
 
37
+ # Whether to use IMAP uid to track last processed message
38
+ config :uid_tracking, :validate => :boolean, :default => false
39
+
40
+ # Path to file with last run time metadata
41
+ config :sincedb_path, :validate => :string, :required => false
42
+
37
43
  def register
38
44
  require "net/imap" # in stdlib
39
45
  require "mail" # gem 'mail'
@@ -50,6 +56,22 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
50
56
  end
51
57
  end
52
58
 
59
+ # Load last processed IMAP uid from file if exists
60
+ if @sincedb_path.nil?
61
+ datapath = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "imap")
62
+ # Ensure that the filepath exists before writing, since it's deeply nested.
63
+ FileUtils::mkdir_p datapath
64
+ @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
65
+ end
66
+ if File.directory?(@sincedb_path)
67
+ raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
68
+ end
69
+ @logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
70
+ if File.exist?(@sincedb_path)
71
+ @uid_last_value = File.read(@sincedb_path).to_i
72
+ @logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
73
+ end
74
+
53
75
  @content_type_re = Regexp.new("^" + @content_type)
54
76
  end # def register
55
77
 
@@ -75,34 +97,56 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
75
97
  # EOFError, OpenSSL::SSL::SSLError
76
98
  imap = connect
77
99
  imap.select(@folder)
78
- ids = imap.search("NOT SEEN")
100
+ if @uid_tracking && @uid_last_value
101
+ # If there are no new messages, uid_search returns @uid_last_value
102
+ # because it is the last message, so we need to delete it.
103
+ ids = imap.uid_search(["UID", (@uid_last_value+1..-1)]).delete_if { |uid|
104
+ uid <= @uid_last_value
105
+ }
106
+ else
107
+ ids = imap.uid_search("NOT SEEN")
108
+ end
79
109
 
80
110
  ids.each_slice(@fetch_count) do |id_set|
81
- items = imap.fetch(id_set, "RFC822")
111
+ items = imap.uid_fetch(id_set, ["BODY.PEEK[]", "UID"])
82
112
  items.each do |item|
83
- next unless item.attr.has_key?("RFC822")
84
- mail = Mail.read_from_string(item.attr["RFC822"])
113
+ next unless item.attr.has_key?("BODY[]")
114
+ mail = Mail.read_from_string(item.attr["BODY[]"])
85
115
  if @strip_attachments
86
116
  queue << parse_mail(mail.without_attachments!)
87
117
  else
88
118
  queue << parse_mail(mail)
89
119
  end
120
+ # Mark message as processed
121
+ @uid_last_value = item.attr["UID"]
122
+ imap.uid_store(@uid_last_value, '+FLAGS', @delete || @expunge ? :Deleted : :Seen)
123
+
124
+ # Stop message processing if it is requested
125
+ break if stop?
90
126
  end
91
127
 
92
- imap.store(id_set, '+FLAGS', @delete ? :Deleted : :Seen)
93
-
94
- end
128
+ # Expunge deleted messages
129
+ imap.expunge() if @expunge
95
130
 
96
- # Enable an 'expunge' IMAP command after the items.each loop
97
- if @expunge
98
- # Force messages to be marked as "Deleted", the above may or may not be working as expected. "Seen" means nothing if you are going to
99
- # delete a message after processing.
100
- imap.store(id_set, '+FLAGS', [:Deleted])
101
- imap.expunge()
131
+ # Stop message fetching if it is requested
132
+ break if stop?
102
133
  end
103
134
 
104
- imap.close
105
- imap.disconnect
135
+ rescue => e
136
+ @logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace)
137
+ # Do not raise error, check_mail will be invoked in the next run time
138
+
139
+ ensure
140
+ # Close the connection (and ignore errors)
141
+ imap.close rescue nil
142
+ imap.disconnect rescue nil
143
+
144
+ # Always save @uid_last_value so when tracking is switched from
145
+ # "NOT SEEN" to "UID" we will continue from first unprocessed message
146
+ if @uid_last_value
147
+ @logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
148
+ File.write(@sincedb_path, @uid_last_value)
149
+ end
106
150
  end
107
151
 
108
152
  def parse_mail(mail)
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-imap'
4
- s.version = '3.0.6'
4
+ s.version = '3.0.7'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Reads mail from an IMAP server"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,7 +25,7 @@ describe LogStash::Inputs::IMAP do
25
25
  allow(imap).to receive(:store)
26
26
  allow(ids).to receive(:each_slice).and_return([])
27
27
 
28
- allow(imap).to receive(:search).with("NOT SEEN").and_return(ids)
28
+ allow(imap).to receive(:uid_search).with("NOT SEEN").and_return(ids)
29
29
  allow(Net::IMAP).to receive(:new).and_return(imap)
30
30
  end
31
31
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-imap
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.6
4
+ version: 3.0.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-06 00:00:00.000000000 Z
11
+ date: 2019-09-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
140
140
  version: '0'
141
141
  requirements: []
142
142
  rubyforge_project:
143
- rubygems_version: 2.6.11
143
+ rubygems_version: 2.6.13
144
144
  signing_key:
145
145
  specification_version: 4
146
146
  summary: Reads mail from an IMAP server