logstash-input-imap 3.0.6 → 3.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +46 -15
- data/lib/logstash/inputs/imap.rb +60 -16
- data/logstash-input-imap.gemspec +1 -1
- data/spec/inputs/imap_spec.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edc2c8598f107e492cecb85f0dc6273b1f4e1147d3042c296131456fa3a1f029
|
4
|
+
data.tar.gz: '09af588e526c930db0a8fe6a5dd86b0965061b81e8ce1b0f49da0912dd06f0a8'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1e0c068137701bf82698a3dd8546164b0723e97fabd317746ef08fe2594ef1f0c0fe7014be1211689a1401df7ccbbefd06f16015e20f336d7b1fce0182853eae
|
7
|
+
data.tar.gz: de968afeb6d1fb8653f307209c2999c71c32f679792354af4a97d1af71758767748b43b21a561edaf82b27dbcbc28bc084244b1cc577a852261121b304b3156f
|
data/CHANGELOG.md
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -45,7 +45,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
45
45
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|Yes
|
46
46
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
|
47
47
|
| <<plugins-{type}s-{plugin}-secure>> |<<boolean,boolean>>|No
|
48
|
+
| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
|
48
49
|
| <<plugins-{type}s-{plugin}-strip_attachments>> |<<boolean,boolean>>|No
|
50
|
+
| <<plugins-{type}s-{plugin}-uid_tracking>> |<<boolean,boolean>>|No
|
49
51
|
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|Yes
|
50
52
|
| <<plugins-{type}s-{plugin}-verify_cert>> |<<boolean,boolean>>|No
|
51
53
|
|=======================================================================
|
@@ -56,7 +58,7 @@ input plugins.
|
|
56
58
|
|
57
59
|
|
58
60
|
[id="plugins-{type}s-{plugin}-check_interval"]
|
59
|
-
===== `check_interval`
|
61
|
+
===== `check_interval`
|
60
62
|
|
61
63
|
* Value type is <<number,number>>
|
62
64
|
* Default value is `300`
|
@@ -64,7 +66,7 @@ input plugins.
|
|
64
66
|
|
65
67
|
|
66
68
|
[id="plugins-{type}s-{plugin}-content_type"]
|
67
|
-
===== `content_type`
|
69
|
+
===== `content_type`
|
68
70
|
|
69
71
|
* Value type is <<string,string>>
|
70
72
|
* Default value is `"text/plain"`
|
@@ -73,7 +75,7 @@ For multipart messages, use the first part that has this
|
|
73
75
|
content-type as the event message.
|
74
76
|
|
75
77
|
[id="plugins-{type}s-{plugin}-delete"]
|
76
|
-
===== `delete`
|
78
|
+
===== `delete`
|
77
79
|
|
78
80
|
* Value type is <<boolean,boolean>>
|
79
81
|
* Default value is `false`
|
@@ -81,7 +83,7 @@ content-type as the event message.
|
|
81
83
|
|
82
84
|
|
83
85
|
[id="plugins-{type}s-{plugin}-expunge"]
|
84
|
-
===== `expunge`
|
86
|
+
===== `expunge`
|
85
87
|
|
86
88
|
* Value type is <<boolean,boolean>>
|
87
89
|
* Default value is `false`
|
@@ -89,7 +91,7 @@ content-type as the event message.
|
|
89
91
|
|
90
92
|
|
91
93
|
[id="plugins-{type}s-{plugin}-fetch_count"]
|
92
|
-
===== `fetch_count`
|
94
|
+
===== `fetch_count`
|
93
95
|
|
94
96
|
* Value type is <<number,number>>
|
95
97
|
* Default value is `50`
|
@@ -97,7 +99,7 @@ content-type as the event message.
|
|
97
99
|
|
98
100
|
|
99
101
|
[id="plugins-{type}s-{plugin}-folder"]
|
100
|
-
===== `folder`
|
102
|
+
===== `folder`
|
101
103
|
|
102
104
|
* Value type is <<string,string>>
|
103
105
|
* Default value is `"INBOX"`
|
@@ -105,7 +107,7 @@ content-type as the event message.
|
|
105
107
|
|
106
108
|
|
107
109
|
[id="plugins-{type}s-{plugin}-host"]
|
108
|
-
===== `host`
|
110
|
+
===== `host`
|
109
111
|
|
110
112
|
* This is a required setting.
|
111
113
|
* Value type is <<string,string>>
|
@@ -114,7 +116,7 @@ content-type as the event message.
|
|
114
116
|
|
115
117
|
|
116
118
|
[id="plugins-{type}s-{plugin}-lowercase_headers"]
|
117
|
-
===== `lowercase_headers`
|
119
|
+
===== `lowercase_headers`
|
118
120
|
|
119
121
|
* Value type is <<boolean,boolean>>
|
120
122
|
* Default value is `true`
|
@@ -122,7 +124,7 @@ content-type as the event message.
|
|
122
124
|
|
123
125
|
|
124
126
|
[id="plugins-{type}s-{plugin}-password"]
|
125
|
-
===== `password`
|
127
|
+
===== `password`
|
126
128
|
|
127
129
|
* This is a required setting.
|
128
130
|
* Value type is <<password,password>>
|
@@ -131,7 +133,7 @@ content-type as the event message.
|
|
131
133
|
|
132
134
|
|
133
135
|
[id="plugins-{type}s-{plugin}-port"]
|
134
|
-
===== `port`
|
136
|
+
===== `port`
|
135
137
|
|
136
138
|
* Value type is <<number,number>>
|
137
139
|
* There is no default value for this setting.
|
@@ -139,23 +141,52 @@ content-type as the event message.
|
|
139
141
|
|
140
142
|
|
141
143
|
[id="plugins-{type}s-{plugin}-secure"]
|
142
|
-
===== `secure`
|
144
|
+
===== `secure`
|
143
145
|
|
144
146
|
* Value type is <<boolean,boolean>>
|
145
147
|
* Default value is `true`
|
146
148
|
|
147
149
|
|
148
150
|
|
151
|
+
[id="plugins-{type}s-{plugin}-sincedb_path"]
|
152
|
+
===== `sincedb_path`
|
153
|
+
|
154
|
+
* Value type is <<string,string>>
|
155
|
+
* There is no default value for this setting.
|
156
|
+
|
157
|
+
Path of the sincedb database file (keeps track of the UID of the last processed
|
158
|
+
mail) that will be written to disk. The default will write sincedb file to
|
159
|
+
`<path.data>/plugins/inputs/imap` directory.
|
160
|
+
NOTE: it must be a file path and not a directory path.
|
161
|
+
|
149
162
|
[id="plugins-{type}s-{plugin}-strip_attachments"]
|
150
|
-
===== `strip_attachments`
|
163
|
+
===== `strip_attachments`
|
151
164
|
|
152
165
|
* Value type is <<boolean,boolean>>
|
153
166
|
* Default value is `false`
|
154
167
|
|
155
168
|
|
156
169
|
|
170
|
+
[id="plugins-{type}s-{plugin}-uid_tracking"]
|
171
|
+
===== `uid_tracking`
|
172
|
+
|
173
|
+
* Value type is <<boolean,boolean>>
|
174
|
+
* Default value is `false`
|
175
|
+
|
176
|
+
When the IMAP input plugin connects to the mailbox for the first time and
|
177
|
+
the UID of the last processed mail is not yet known, the unread mails are
|
178
|
+
first downloaded and the UID of the last processed mail is saved. From
|
179
|
+
this point on, if `uid_tracking` is set to `true`, all new mail will be
|
180
|
+
downloaded regardless of whether they are marked as read or unread. This
|
181
|
+
allows users or other services to use the mailbox simultaneously with the
|
182
|
+
IMAP input plugin. UID of the last processed mail is always saved regardles
|
183
|
+
of the `uid_tracking` value, so you can switch its value as needed. In
|
184
|
+
transition from the previous IMAP input plugin version, first process at least
|
185
|
+
one mail with `uid_tracking` set to `false` to save the UID of the last
|
186
|
+
processed mail and then switch `uid_tracking` to `true`.
|
187
|
+
|
157
188
|
[id="plugins-{type}s-{plugin}-user"]
|
158
|
-
===== `user`
|
189
|
+
===== `user`
|
159
190
|
|
160
191
|
* This is a required setting.
|
161
192
|
* Value type is <<string,string>>
|
@@ -164,7 +195,7 @@ content-type as the event message.
|
|
164
195
|
|
165
196
|
|
166
197
|
[id="plugins-{type}s-{plugin}-verify_cert"]
|
167
|
-
===== `verify_cert`
|
198
|
+
===== `verify_cert`
|
168
199
|
|
169
200
|
* Value type is <<boolean,boolean>>
|
170
201
|
* Default value is `true`
|
@@ -176,4 +207,4 @@ content-type as the event message.
|
|
176
207
|
[id="plugins-{type}s-{plugin}-common-options"]
|
177
208
|
include::{include_path}/{type}.asciidoc[]
|
178
209
|
|
179
|
-
:default_codec!:
|
210
|
+
:default_codec!:
|
data/lib/logstash/inputs/imap.rb
CHANGED
@@ -29,11 +29,17 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
|
|
29
29
|
config :delete, :validate => :boolean, :default => false
|
30
30
|
config :expunge, :validate => :boolean, :default => false
|
31
31
|
config :strip_attachments, :validate => :boolean, :default => false
|
32
|
-
|
32
|
+
|
33
33
|
# For multipart messages, use the first part that has this
|
34
34
|
# content-type as the event message.
|
35
35
|
config :content_type, :validate => :string, :default => "text/plain"
|
36
36
|
|
37
|
+
# Whether to use IMAP uid to track last processed message
|
38
|
+
config :uid_tracking, :validate => :boolean, :default => false
|
39
|
+
|
40
|
+
# Path to file with last run time metadata
|
41
|
+
config :sincedb_path, :validate => :string, :required => false
|
42
|
+
|
37
43
|
def register
|
38
44
|
require "net/imap" # in stdlib
|
39
45
|
require "mail" # gem 'mail'
|
@@ -50,6 +56,22 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
|
|
50
56
|
end
|
51
57
|
end
|
52
58
|
|
59
|
+
# Load last processed IMAP uid from file if exists
|
60
|
+
if @sincedb_path.nil?
|
61
|
+
datapath = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "imap")
|
62
|
+
# Ensure that the filepath exists before writing, since it's deeply nested.
|
63
|
+
FileUtils::mkdir_p datapath
|
64
|
+
@sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
|
65
|
+
end
|
66
|
+
if File.directory?(@sincedb_path)
|
67
|
+
raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
|
68
|
+
end
|
69
|
+
@logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
|
70
|
+
if File.exist?(@sincedb_path)
|
71
|
+
@uid_last_value = File.read(@sincedb_path).to_i
|
72
|
+
@logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
|
73
|
+
end
|
74
|
+
|
53
75
|
@content_type_re = Regexp.new("^" + @content_type)
|
54
76
|
end # def register
|
55
77
|
|
@@ -75,34 +97,56 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
|
|
75
97
|
# EOFError, OpenSSL::SSL::SSLError
|
76
98
|
imap = connect
|
77
99
|
imap.select(@folder)
|
78
|
-
|
100
|
+
if @uid_tracking && @uid_last_value
|
101
|
+
# If there are no new messages, uid_search returns @uid_last_value
|
102
|
+
# because it is the last message, so we need to delete it.
|
103
|
+
ids = imap.uid_search(["UID", (@uid_last_value+1..-1)]).delete_if { |uid|
|
104
|
+
uid <= @uid_last_value
|
105
|
+
}
|
106
|
+
else
|
107
|
+
ids = imap.uid_search("NOT SEEN")
|
108
|
+
end
|
79
109
|
|
80
110
|
ids.each_slice(@fetch_count) do |id_set|
|
81
|
-
items = imap.
|
111
|
+
items = imap.uid_fetch(id_set, ["BODY.PEEK[]", "UID"])
|
82
112
|
items.each do |item|
|
83
|
-
next unless item.attr.has_key?("
|
84
|
-
mail = Mail.read_from_string(item.attr["
|
113
|
+
next unless item.attr.has_key?("BODY[]")
|
114
|
+
mail = Mail.read_from_string(item.attr["BODY[]"])
|
85
115
|
if @strip_attachments
|
86
116
|
queue << parse_mail(mail.without_attachments!)
|
87
117
|
else
|
88
118
|
queue << parse_mail(mail)
|
89
119
|
end
|
120
|
+
# Mark message as processed
|
121
|
+
@uid_last_value = item.attr["UID"]
|
122
|
+
imap.uid_store(@uid_last_value, '+FLAGS', @delete || @expunge ? :Deleted : :Seen)
|
123
|
+
|
124
|
+
# Stop message processing if it is requested
|
125
|
+
break if stop?
|
90
126
|
end
|
91
127
|
|
92
|
-
|
93
|
-
|
94
|
-
end
|
128
|
+
# Expunge deleted messages
|
129
|
+
imap.expunge() if @expunge
|
95
130
|
|
96
|
-
|
97
|
-
|
98
|
-
# Force messages to be marked as "Deleted", the above may or may not be working as expected. "Seen" means nothing if you are going to
|
99
|
-
# delete a message after processing.
|
100
|
-
imap.store(id_set, '+FLAGS', [:Deleted])
|
101
|
-
imap.expunge()
|
131
|
+
# Stop message fetching if it is requested
|
132
|
+
break if stop?
|
102
133
|
end
|
103
134
|
|
104
|
-
|
105
|
-
|
135
|
+
rescue => e
|
136
|
+
@logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace)
|
137
|
+
# Do not raise error, check_mail will be invoked in the next run time
|
138
|
+
|
139
|
+
ensure
|
140
|
+
# Close the connection (and ignore errors)
|
141
|
+
imap.close rescue nil
|
142
|
+
imap.disconnect rescue nil
|
143
|
+
|
144
|
+
# Always save @uid_last_value so when tracking is switched from
|
145
|
+
# "NOT SEEN" to "UID" we will continue from first unprocessed message
|
146
|
+
if @uid_last_value
|
147
|
+
@logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
|
148
|
+
File.write(@sincedb_path, @uid_last_value)
|
149
|
+
end
|
106
150
|
end
|
107
151
|
|
108
152
|
def parse_mail(mail)
|
data/logstash-input-imap.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-imap'
|
4
|
-
s.version = '3.0.
|
4
|
+
s.version = '3.0.7'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads mail from an IMAP server"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/imap_spec.rb
CHANGED
@@ -25,7 +25,7 @@ describe LogStash::Inputs::IMAP do
|
|
25
25
|
allow(imap).to receive(:store)
|
26
26
|
allow(ids).to receive(:each_slice).and_return([])
|
27
27
|
|
28
|
-
allow(imap).to receive(:
|
28
|
+
allow(imap).to receive(:uid_search).with("NOT SEEN").and_return(ids)
|
29
29
|
allow(Net::IMAP).to receive(:new).and_return(imap)
|
30
30
|
end
|
31
31
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-imap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
142
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.6.
|
143
|
+
rubygems_version: 2.6.13
|
144
144
|
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: Reads mail from an IMAP server
|