logstash-input-imap 3.0.6 → 3.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +46 -15
- data/lib/logstash/inputs/imap.rb +60 -16
- data/logstash-input-imap.gemspec +1 -1
- data/spec/inputs/imap_spec.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edc2c8598f107e492cecb85f0dc6273b1f4e1147d3042c296131456fa3a1f029
|
4
|
+
data.tar.gz: '09af588e526c930db0a8fe6a5dd86b0965061b81e8ce1b0f49da0912dd06f0a8'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1e0c068137701bf82698a3dd8546164b0723e97fabd317746ef08fe2594ef1f0c0fe7014be1211689a1401df7ccbbefd06f16015e20f336d7b1fce0182853eae
|
7
|
+
data.tar.gz: de968afeb6d1fb8653f307209c2999c71c32f679792354af4a97d1af71758767748b43b21a561edaf82b27dbcbc28bc084244b1cc577a852261121b304b3156f
|
data/CHANGELOG.md
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -45,7 +45,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
45
45
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|Yes
|
46
46
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
|
47
47
|
| <<plugins-{type}s-{plugin}-secure>> |<<boolean,boolean>>|No
|
48
|
+
| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
|
48
49
|
| <<plugins-{type}s-{plugin}-strip_attachments>> |<<boolean,boolean>>|No
|
50
|
+
| <<plugins-{type}s-{plugin}-uid_tracking>> |<<boolean,boolean>>|No
|
49
51
|
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|Yes
|
50
52
|
| <<plugins-{type}s-{plugin}-verify_cert>> |<<boolean,boolean>>|No
|
51
53
|
|=======================================================================
|
@@ -56,7 +58,7 @@ input plugins.
|
|
56
58
|
|
57
59
|
|
58
60
|
[id="plugins-{type}s-{plugin}-check_interval"]
|
59
|
-
===== `check_interval`
|
61
|
+
===== `check_interval`
|
60
62
|
|
61
63
|
* Value type is <<number,number>>
|
62
64
|
* Default value is `300`
|
@@ -64,7 +66,7 @@ input plugins.
|
|
64
66
|
|
65
67
|
|
66
68
|
[id="plugins-{type}s-{plugin}-content_type"]
|
67
|
-
===== `content_type`
|
69
|
+
===== `content_type`
|
68
70
|
|
69
71
|
* Value type is <<string,string>>
|
70
72
|
* Default value is `"text/plain"`
|
@@ -73,7 +75,7 @@ For multipart messages, use the first part that has this
|
|
73
75
|
content-type as the event message.
|
74
76
|
|
75
77
|
[id="plugins-{type}s-{plugin}-delete"]
|
76
|
-
===== `delete`
|
78
|
+
===== `delete`
|
77
79
|
|
78
80
|
* Value type is <<boolean,boolean>>
|
79
81
|
* Default value is `false`
|
@@ -81,7 +83,7 @@ content-type as the event message.
|
|
81
83
|
|
82
84
|
|
83
85
|
[id="plugins-{type}s-{plugin}-expunge"]
|
84
|
-
===== `expunge`
|
86
|
+
===== `expunge`
|
85
87
|
|
86
88
|
* Value type is <<boolean,boolean>>
|
87
89
|
* Default value is `false`
|
@@ -89,7 +91,7 @@ content-type as the event message.
|
|
89
91
|
|
90
92
|
|
91
93
|
[id="plugins-{type}s-{plugin}-fetch_count"]
|
92
|
-
===== `fetch_count`
|
94
|
+
===== `fetch_count`
|
93
95
|
|
94
96
|
* Value type is <<number,number>>
|
95
97
|
* Default value is `50`
|
@@ -97,7 +99,7 @@ content-type as the event message.
|
|
97
99
|
|
98
100
|
|
99
101
|
[id="plugins-{type}s-{plugin}-folder"]
|
100
|
-
===== `folder`
|
102
|
+
===== `folder`
|
101
103
|
|
102
104
|
* Value type is <<string,string>>
|
103
105
|
* Default value is `"INBOX"`
|
@@ -105,7 +107,7 @@ content-type as the event message.
|
|
105
107
|
|
106
108
|
|
107
109
|
[id="plugins-{type}s-{plugin}-host"]
|
108
|
-
===== `host`
|
110
|
+
===== `host`
|
109
111
|
|
110
112
|
* This is a required setting.
|
111
113
|
* Value type is <<string,string>>
|
@@ -114,7 +116,7 @@ content-type as the event message.
|
|
114
116
|
|
115
117
|
|
116
118
|
[id="plugins-{type}s-{plugin}-lowercase_headers"]
|
117
|
-
===== `lowercase_headers`
|
119
|
+
===== `lowercase_headers`
|
118
120
|
|
119
121
|
* Value type is <<boolean,boolean>>
|
120
122
|
* Default value is `true`
|
@@ -122,7 +124,7 @@ content-type as the event message.
|
|
122
124
|
|
123
125
|
|
124
126
|
[id="plugins-{type}s-{plugin}-password"]
|
125
|
-
===== `password`
|
127
|
+
===== `password`
|
126
128
|
|
127
129
|
* This is a required setting.
|
128
130
|
* Value type is <<password,password>>
|
@@ -131,7 +133,7 @@ content-type as the event message.
|
|
131
133
|
|
132
134
|
|
133
135
|
[id="plugins-{type}s-{plugin}-port"]
|
134
|
-
===== `port`
|
136
|
+
===== `port`
|
135
137
|
|
136
138
|
* Value type is <<number,number>>
|
137
139
|
* There is no default value for this setting.
|
@@ -139,23 +141,52 @@ content-type as the event message.
|
|
139
141
|
|
140
142
|
|
141
143
|
[id="plugins-{type}s-{plugin}-secure"]
|
142
|
-
===== `secure`
|
144
|
+
===== `secure`
|
143
145
|
|
144
146
|
* Value type is <<boolean,boolean>>
|
145
147
|
* Default value is `true`
|
146
148
|
|
147
149
|
|
148
150
|
|
151
|
+
[id="plugins-{type}s-{plugin}-sincedb_path"]
|
152
|
+
===== `sincedb_path`
|
153
|
+
|
154
|
+
* Value type is <<string,string>>
|
155
|
+
* There is no default value for this setting.
|
156
|
+
|
157
|
+
Path of the sincedb database file (keeps track of the UID of the last processed
|
158
|
+
mail) that will be written to disk. The default will write sincedb file to
|
159
|
+
`<path.data>/plugins/inputs/imap` directory.
|
160
|
+
NOTE: it must be a file path and not a directory path.
|
161
|
+
|
149
162
|
[id="plugins-{type}s-{plugin}-strip_attachments"]
|
150
|
-
===== `strip_attachments`
|
163
|
+
===== `strip_attachments`
|
151
164
|
|
152
165
|
* Value type is <<boolean,boolean>>
|
153
166
|
* Default value is `false`
|
154
167
|
|
155
168
|
|
156
169
|
|
170
|
+
[id="plugins-{type}s-{plugin}-uid_tracking"]
|
171
|
+
===== `uid_tracking`
|
172
|
+
|
173
|
+
* Value type is <<boolean,boolean>>
|
174
|
+
* Default value is `false`
|
175
|
+
|
176
|
+
When the IMAP input plugin connects to the mailbox for the first time and
|
177
|
+
the UID of the last processed mail is not yet known, the unread mails are
|
178
|
+
first downloaded and the UID of the last processed mail is saved. From
|
179
|
+
this point on, if `uid_tracking` is set to `true`, all new mail will be
|
180
|
+
downloaded regardless of whether they are marked as read or unread. This
|
181
|
+
allows users or other services to use the mailbox simultaneously with the
|
182
|
+
IMAP input plugin. UID of the last processed mail is always saved regardles
|
183
|
+
of the `uid_tracking` value, so you can switch its value as needed. In
|
184
|
+
transition from the previous IMAP input plugin version, first process at least
|
185
|
+
one mail with `uid_tracking` set to `false` to save the UID of the last
|
186
|
+
processed mail and then switch `uid_tracking` to `true`.
|
187
|
+
|
157
188
|
[id="plugins-{type}s-{plugin}-user"]
|
158
|
-
===== `user`
|
189
|
+
===== `user`
|
159
190
|
|
160
191
|
* This is a required setting.
|
161
192
|
* Value type is <<string,string>>
|
@@ -164,7 +195,7 @@ content-type as the event message.
|
|
164
195
|
|
165
196
|
|
166
197
|
[id="plugins-{type}s-{plugin}-verify_cert"]
|
167
|
-
===== `verify_cert`
|
198
|
+
===== `verify_cert`
|
168
199
|
|
169
200
|
* Value type is <<boolean,boolean>>
|
170
201
|
* Default value is `true`
|
@@ -176,4 +207,4 @@ content-type as the event message.
|
|
176
207
|
[id="plugins-{type}s-{plugin}-common-options"]
|
177
208
|
include::{include_path}/{type}.asciidoc[]
|
178
209
|
|
179
|
-
:default_codec!:
|
210
|
+
:default_codec!:
|
data/lib/logstash/inputs/imap.rb
CHANGED
@@ -29,11 +29,17 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
|
|
29
29
|
config :delete, :validate => :boolean, :default => false
|
30
30
|
config :expunge, :validate => :boolean, :default => false
|
31
31
|
config :strip_attachments, :validate => :boolean, :default => false
|
32
|
-
|
32
|
+
|
33
33
|
# For multipart messages, use the first part that has this
|
34
34
|
# content-type as the event message.
|
35
35
|
config :content_type, :validate => :string, :default => "text/plain"
|
36
36
|
|
37
|
+
# Whether to use IMAP uid to track last processed message
|
38
|
+
config :uid_tracking, :validate => :boolean, :default => false
|
39
|
+
|
40
|
+
# Path to file with last run time metadata
|
41
|
+
config :sincedb_path, :validate => :string, :required => false
|
42
|
+
|
37
43
|
def register
|
38
44
|
require "net/imap" # in stdlib
|
39
45
|
require "mail" # gem 'mail'
|
@@ -50,6 +56,22 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
|
|
50
56
|
end
|
51
57
|
end
|
52
58
|
|
59
|
+
# Load last processed IMAP uid from file if exists
|
60
|
+
if @sincedb_path.nil?
|
61
|
+
datapath = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "imap")
|
62
|
+
# Ensure that the filepath exists before writing, since it's deeply nested.
|
63
|
+
FileUtils::mkdir_p datapath
|
64
|
+
@sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
|
65
|
+
end
|
66
|
+
if File.directory?(@sincedb_path)
|
67
|
+
raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
|
68
|
+
end
|
69
|
+
@logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
|
70
|
+
if File.exist?(@sincedb_path)
|
71
|
+
@uid_last_value = File.read(@sincedb_path).to_i
|
72
|
+
@logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
|
73
|
+
end
|
74
|
+
|
53
75
|
@content_type_re = Regexp.new("^" + @content_type)
|
54
76
|
end # def register
|
55
77
|
|
@@ -75,34 +97,56 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
|
|
75
97
|
# EOFError, OpenSSL::SSL::SSLError
|
76
98
|
imap = connect
|
77
99
|
imap.select(@folder)
|
78
|
-
|
100
|
+
if @uid_tracking && @uid_last_value
|
101
|
+
# If there are no new messages, uid_search returns @uid_last_value
|
102
|
+
# because it is the last message, so we need to delete it.
|
103
|
+
ids = imap.uid_search(["UID", (@uid_last_value+1..-1)]).delete_if { |uid|
|
104
|
+
uid <= @uid_last_value
|
105
|
+
}
|
106
|
+
else
|
107
|
+
ids = imap.uid_search("NOT SEEN")
|
108
|
+
end
|
79
109
|
|
80
110
|
ids.each_slice(@fetch_count) do |id_set|
|
81
|
-
items = imap.
|
111
|
+
items = imap.uid_fetch(id_set, ["BODY.PEEK[]", "UID"])
|
82
112
|
items.each do |item|
|
83
|
-
next unless item.attr.has_key?("
|
84
|
-
mail = Mail.read_from_string(item.attr["
|
113
|
+
next unless item.attr.has_key?("BODY[]")
|
114
|
+
mail = Mail.read_from_string(item.attr["BODY[]"])
|
85
115
|
if @strip_attachments
|
86
116
|
queue << parse_mail(mail.without_attachments!)
|
87
117
|
else
|
88
118
|
queue << parse_mail(mail)
|
89
119
|
end
|
120
|
+
# Mark message as processed
|
121
|
+
@uid_last_value = item.attr["UID"]
|
122
|
+
imap.uid_store(@uid_last_value, '+FLAGS', @delete || @expunge ? :Deleted : :Seen)
|
123
|
+
|
124
|
+
# Stop message processing if it is requested
|
125
|
+
break if stop?
|
90
126
|
end
|
91
127
|
|
92
|
-
|
93
|
-
|
94
|
-
end
|
128
|
+
# Expunge deleted messages
|
129
|
+
imap.expunge() if @expunge
|
95
130
|
|
96
|
-
|
97
|
-
|
98
|
-
# Force messages to be marked as "Deleted", the above may or may not be working as expected. "Seen" means nothing if you are going to
|
99
|
-
# delete a message after processing.
|
100
|
-
imap.store(id_set, '+FLAGS', [:Deleted])
|
101
|
-
imap.expunge()
|
131
|
+
# Stop message fetching if it is requested
|
132
|
+
break if stop?
|
102
133
|
end
|
103
134
|
|
104
|
-
|
105
|
-
|
135
|
+
rescue => e
|
136
|
+
@logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace)
|
137
|
+
# Do not raise error, check_mail will be invoked in the next run time
|
138
|
+
|
139
|
+
ensure
|
140
|
+
# Close the connection (and ignore errors)
|
141
|
+
imap.close rescue nil
|
142
|
+
imap.disconnect rescue nil
|
143
|
+
|
144
|
+
# Always save @uid_last_value so when tracking is switched from
|
145
|
+
# "NOT SEEN" to "UID" we will continue from first unprocessed message
|
146
|
+
if @uid_last_value
|
147
|
+
@logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
|
148
|
+
File.write(@sincedb_path, @uid_last_value)
|
149
|
+
end
|
106
150
|
end
|
107
151
|
|
108
152
|
def parse_mail(mail)
|
data/logstash-input-imap.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-imap'
|
4
|
-
s.version = '3.0.
|
4
|
+
s.version = '3.0.7'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads mail from an IMAP server"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/imap_spec.rb
CHANGED
@@ -25,7 +25,7 @@ describe LogStash::Inputs::IMAP do
|
|
25
25
|
allow(imap).to receive(:store)
|
26
26
|
allow(ids).to receive(:each_slice).and_return([])
|
27
27
|
|
28
|
-
allow(imap).to receive(:
|
28
|
+
allow(imap).to receive(:uid_search).with("NOT SEEN").and_return(ids)
|
29
29
|
allow(Net::IMAP).to receive(:new).and_return(imap)
|
30
30
|
end
|
31
31
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-imap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
142
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.6.
|
143
|
+
rubygems_version: 2.6.13
|
144
144
|
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: Reads mail from an IMAP server
|