logstash-filter-date 2.1.2 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/filters/date.rb +86 -22
- data/logstash-filter-date.gemspec +2 -2
- data/spec/filters/date_spec.rb +126 -0
- metadata +29 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ea6c51877b6a28e814bd00e623932612e5b9f8f1
|
4
|
+
data.tar.gz: ebe269a2eb0210e9b4eedcb796ecedc06bb830e4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d905fb89de9f982b6b2125bab4adfc68861d17dcfd4d0bb92b3497382681964f29c82e7e26ee4930fb83aaa13929dcc8a54daac7c5063f430eca182337138ad
|
7
|
+
data.tar.gz: 75b71d3e79ac533ab6ad51d4b2ee749847d841f0601a9f8db44577b70941e4df30c534df9996e7ec6b220d5c556e94e11eb3457c001017ec4edef7cc32c232db
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
# 2.1.4
|
2
|
+
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
3
|
+
# 2.1.3
|
4
|
+
- New dependency requirements for logstash-core for the 5.0 release
|
1
5
|
## 2.1.2
|
2
6
|
- Make tests less reliant on implementation details of LogStash::Event
|
3
7
|
## 2.1.1
|
@@ -48,11 +48,6 @@ class LogStash::Filters::Date < LogStash::Filters::Base
|
|
48
48
|
# an english parser will also be used as a fallback mechanism.
|
49
49
|
config :locale, :validate => :string
|
50
50
|
|
51
|
-
# The date formats allowed are anything allowed by Joda-Time (java time
|
52
|
-
# library). You can see the docs for this format here:
|
53
|
-
#
|
54
|
-
# http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html[joda.time.format.DateTimeFormat]
|
55
|
-
#
|
56
51
|
# An array with field name first, and format patterns following, `[ field,
|
57
52
|
# formats... ]`
|
58
53
|
#
|
@@ -84,6 +79,71 @@ class LogStash::Filters::Date < LogStash::Filters::Base
|
|
84
79
|
# If your field is nested in your structure, you can use the nested
|
85
80
|
# syntax `[foo][bar]` to match its value. For more information, please refer to
|
86
81
|
# <<logstash-config-field-references>>
|
82
|
+
#
|
83
|
+
# *More details on the syntax*
|
84
|
+
#
|
85
|
+
# The syntax used for parsing date and time text uses letters to indicate the
|
86
|
+
# kind of time value (month, minute, etc), and a repetition of letters to
|
87
|
+
# indicate the form of that value (2-digit month, full month name, etc).
|
88
|
+
#
|
89
|
+
# Here's what you can use to parse dates and times:
|
90
|
+
#
|
91
|
+
# [horizontal]
|
92
|
+
# y:: year
|
93
|
+
# yyyy::: full year number. Example: `2015`.
|
94
|
+
# yy::: two-digit year. Example: `15` for the year 2015.
|
95
|
+
#
|
96
|
+
# M:: month of the year
|
97
|
+
# M::: minimal-digit month. Example: `1` for January and `12` for December.
|
98
|
+
# MM::: two-digit month. zero-padded if needed. Example: `01` for January and `12` for December
|
99
|
+
# MMM::: abbreviated month text. Example: `Jan` for January. Note: The language used depends on your locale. See the `locale` setting for how to change the language.
|
100
|
+
# MMMM::: full month text, Example: `January`. Note: The language used depends on your locale.
|
101
|
+
#
|
102
|
+
# d:: day of the month
|
103
|
+
# d::: minimal-digit day. Example: `1` for the 1st of the month.
|
104
|
+
# dd::: two-digit day, zero-padded if needed. Example: `01` for the 1st of the month.
|
105
|
+
#
|
106
|
+
# H:: hour of the day (24-hour clock)
|
107
|
+
# H::: minimal-digit hour. Example: `0` for midnight.
|
108
|
+
# HH::: two-digit hour, zero-padded if needed. Example: `00` for midnight.
|
109
|
+
#
|
110
|
+
# m:: minutes of the hour (60 minutes per hour)
|
111
|
+
# m::: minimal-digit minutes. Example: `0`.
|
112
|
+
# mm::: two-digit minutes, zero-padded if needed. Example: `00`.
|
113
|
+
#
|
114
|
+
# s:: seconds of the minute (60 seconds per minute)
|
115
|
+
# s::: minimal-digit seconds. Example: `0`.
|
116
|
+
# ss::: two-digit seconds, zero-padded if needed. Example: `00`.
|
117
|
+
#
|
118
|
+
# S:: fraction of a second
|
119
|
+
# *Maximum precision is milliseconds (`SSS`). Beyond that, zeroes are appended.*
|
120
|
+
# S::: tenths of a second. Example: `0` for a subsecond value `012`
|
121
|
+
# SS::: hundredths of a second. Example: `01` for a subsecond value `01`
|
122
|
+
# SSS::: thousandths of a second. Example: `012` for a subsecond value `012`
|
123
|
+
#
|
124
|
+
# Z:: time zone offset or identity
|
125
|
+
# Z::: Timezone offset structured as HHmm (hour and minutes offset from Zulu/UTC). Example: `-0700`.
|
126
|
+
# ZZ::: Timezone offset structured as HH:mm (colon in between hour and minute offsets). Example: `-07:00`.
|
127
|
+
# ZZZ::: Timezone identity. Example: `America/Los_Angeles`. Note: Valid IDs are listed on the http://joda-time.sourceforge.net/timezones.html[Joda.org available time zones page].
|
128
|
+
#
|
129
|
+
# z:: time zone names. *Time zone names ('z') cannot be parsed.*
|
130
|
+
#
|
131
|
+
# w:: week of the year
|
132
|
+
# w::: minimal-digit week. Example: `1`.
|
133
|
+
# ww::: two-digit week, zero-padded if needed. Example: `01`.
|
134
|
+
#
|
135
|
+
# D:: day of the year
|
136
|
+
#
|
137
|
+
# e:: day of the week (number)
|
138
|
+
#
|
139
|
+
# E:: day of the week (text)
|
140
|
+
# E, EE, EEE::: Abbreviated day of the week. Example: `Mon`, `Tue`, `Wed`, `Thu`, `Fri`, `Sat`, `Sun`. Note: The actual language of this will depend on your locale.
|
141
|
+
# EEEE::: The full text day of the week. Example: `Monday`, `Tuesday`, ... Note: The actual language of this will depend on your locale.
|
142
|
+
#
|
143
|
+
# For non-formatting syntax, you'll need to put single-quote characters around the value. For example, if you were parsing ISO8601 time, "2015-01-01T01:12:23" that little "T" isn't a valid time format, and you want to say "literally, a T", your format would be this: "yyyy-MM-dd'T'HH:mm:ss"
|
144
|
+
#
|
145
|
+
# Other less common date units, such as era (G), century \(C), am/pm (a), and # more, can be learned about on the
|
146
|
+
# http://www.joda.org/joda-time/key_format.html[joda-time documentation].
|
87
147
|
config :match, :validate => :array, :default => []
|
88
148
|
|
89
149
|
# Store the matching timestamp into the given target field. If not provided,
|
@@ -125,6 +185,24 @@ class LogStash::Filters::Date < LogStash::Filters::Base
|
|
125
185
|
setupMatcher(@config["match"].shift, locale, @config["match"] )
|
126
186
|
end
|
127
187
|
|
188
|
+
def parseWithJodaParser(joda_parser, date, format_has_year)
|
189
|
+
return joda_parser.parseMillis(date) if format_has_year
|
190
|
+
now = Time.now
|
191
|
+
now_month = now.month
|
192
|
+
result = joda_parser.parseDateTime(date)
|
193
|
+
event_month = result.getMonthOfYear
|
194
|
+
|
195
|
+
if (event_month == now_month)
|
196
|
+
result.with_year(now.year)
|
197
|
+
elsif (event_month == 12 && now_month == 1)
|
198
|
+
result.with_year(now.year-1)
|
199
|
+
elsif (event_month == 1 && now_month == 12)
|
200
|
+
result.with_year(now.year+1)
|
201
|
+
else
|
202
|
+
result.with_year(now.year)
|
203
|
+
end.get_millis
|
204
|
+
end
|
205
|
+
|
128
206
|
def setupMatcher(field, locale, value)
|
129
207
|
value.each do |format|
|
130
208
|
parsers = []
|
@@ -181,25 +259,11 @@ class LogStash::Filters::Date < LogStash::Filters::Base
|
|
181
259
|
end
|
182
260
|
if @sprintf_timezone
|
183
261
|
parsers << lambda { |date , tz|
|
184
|
-
joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz))
|
262
|
+
return parseWithJodaParser(joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)), date, format_has_year)
|
185
263
|
}
|
186
264
|
end
|
187
265
|
parsers << lambda do |date|
|
188
|
-
return joda_parser
|
189
|
-
now = Time.now
|
190
|
-
now_month = now.month
|
191
|
-
result = joda_parser.parseDateTime(date)
|
192
|
-
event_month = result.getMonthOfYear
|
193
|
-
|
194
|
-
if (event_month == now_month)
|
195
|
-
result.with_year(now.year)
|
196
|
-
elsif (event_month == 12 && now_month == 1)
|
197
|
-
result.with_year(now.year-1)
|
198
|
-
elsif (event_month == 1 && now_month == 12)
|
199
|
-
result.with_year(now.year+1)
|
200
|
-
else
|
201
|
-
result.with_year(now.year)
|
202
|
-
end.get_millis
|
266
|
+
return parseWithJodaParser(joda_parser, date, format_has_year)
|
203
267
|
end
|
204
268
|
|
205
269
|
#Include a fallback parser to english when default locale is non-english
|
@@ -207,7 +271,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
|
|
207
271
|
"en" != java.util.Locale.getDefault().getLanguage() &&
|
208
272
|
(format.include?("MMM") || format.include?("E"))
|
209
273
|
en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
|
210
|
-
parsers << lambda { |date| en_joda_parser
|
274
|
+
parsers << lambda { |date| parseWithJodaParser(en_joda_parser, date, format_has_year) }
|
211
275
|
end
|
212
276
|
rescue JavaException => e
|
213
277
|
raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-date'
|
4
|
-
s.version = '2.1.
|
4
|
+
s.version = '2.1.4'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core", "
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
|
24
24
|
s.add_runtime_dependency 'logstash-input-generator'
|
25
25
|
s.add_runtime_dependency 'logstash-codec-json'
|
26
26
|
s.add_runtime_dependency 'logstash-output-null'
|
data/spec/filters/date_spec.rb
CHANGED
@@ -386,6 +386,67 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
|
|
386
386
|
end # times.each
|
387
387
|
end
|
388
388
|
|
389
|
+
context "Default year handling when parsing with timezone from event" do
|
390
|
+
|
391
|
+
describe "LOGSTASH-34 - Default year should be this year" do
|
392
|
+
config <<-CONFIG
|
393
|
+
filter {
|
394
|
+
date {
|
395
|
+
match => [ "message", "EEE MMM dd HH:mm:ss" ]
|
396
|
+
locale => "en"
|
397
|
+
timezone => "%{mytz}"
|
398
|
+
}
|
399
|
+
}
|
400
|
+
CONFIG
|
401
|
+
|
402
|
+
sample("message" => "Sun Jun 02 20:38:03", "mytz" => "UTC") do
|
403
|
+
insist { subject["@timestamp"].year } == Time.now.year
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
describe "fill last year if december events arrive in january" do
|
408
|
+
config <<-CONFIG
|
409
|
+
filter {
|
410
|
+
date {
|
411
|
+
match => [ "message", "MMM dd HH:mm:ss" ]
|
412
|
+
locale => "en"
|
413
|
+
timezone => "%{mytz}"
|
414
|
+
}
|
415
|
+
}
|
416
|
+
CONFIG
|
417
|
+
|
418
|
+
before(:each) do
|
419
|
+
logstash_time = Time.utc(2014,1,1,00,30,50)
|
420
|
+
allow(Time).to receive(:now).and_return(logstash_time)
|
421
|
+
end
|
422
|
+
|
423
|
+
sample("message" => "Dec 31 23:59:00", "mytz" => "UTC") do
|
424
|
+
insist { subject["@timestamp"].year } == 2013
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
describe "fill next year if january events arrive in december" do
|
429
|
+
config <<-CONFIG
|
430
|
+
filter {
|
431
|
+
date {
|
432
|
+
match => [ "message", "MMM dd HH:mm:ss" ]
|
433
|
+
locale => "en"
|
434
|
+
timezone => "%{mytz}"
|
435
|
+
}
|
436
|
+
}
|
437
|
+
CONFIG
|
438
|
+
|
439
|
+
before(:each) do
|
440
|
+
logstash_time = Time.utc(2013,12,31,23,59,50)
|
441
|
+
allow(Time).to receive(:now).and_return(logstash_time)
|
442
|
+
end
|
443
|
+
|
444
|
+
sample( "message" => "Jan 01 01:00:00", "mytz" => "UTC") do
|
445
|
+
insist { subject["@timestamp"].year } == 2014
|
446
|
+
end
|
447
|
+
end
|
448
|
+
end
|
449
|
+
|
389
450
|
describe "LOGSTASH-34 - Default year should be this year" do
|
390
451
|
config <<-CONFIG
|
391
452
|
filter {
|
@@ -526,4 +587,69 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
|
|
526
587
|
#Restore default locale
|
527
588
|
java.util.Locale.setDefault(default_locale)
|
528
589
|
end
|
590
|
+
|
591
|
+
context "Default year handling when parsing with english fallback parser" do
|
592
|
+
|
593
|
+
before(:each) do
|
594
|
+
default_locale = java.util.Locale.getDefault()
|
595
|
+
#Override default locale with non-english
|
596
|
+
java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
|
597
|
+
end
|
598
|
+
|
599
|
+
puts "override locale"
|
600
|
+
describe "LOGSTASH-34 - Default year should be this year" do
|
601
|
+
config <<-CONFIG
|
602
|
+
filter {
|
603
|
+
date {
|
604
|
+
match => [ "message", "EEE MMM dd HH:mm:ss" ]
|
605
|
+
timezone => "UTC"
|
606
|
+
}
|
607
|
+
}
|
608
|
+
CONFIG
|
609
|
+
|
610
|
+
sample "Sun Jun 02 20:38:03" do
|
611
|
+
insist { subject["@timestamp"].year } == Time.now.year
|
612
|
+
end
|
613
|
+
end
|
614
|
+
|
615
|
+
describe "fill last year if december events arrive in january" do
|
616
|
+
config <<-CONFIG
|
617
|
+
filter {
|
618
|
+
date {
|
619
|
+
match => [ "message", "MMM dd HH:mm:ss" ]
|
620
|
+
timezone => "UTC"
|
621
|
+
}
|
622
|
+
}
|
623
|
+
CONFIG
|
624
|
+
|
625
|
+
before(:each) do
|
626
|
+
logstash_time = Time.utc(2014,1,1,00,30,50)
|
627
|
+
allow(Time).to receive(:now).and_return(logstash_time)
|
628
|
+
end
|
629
|
+
|
630
|
+
sample "Dec 31 23:59:00" do
|
631
|
+
insist { subject["@timestamp"].year } == 2013
|
632
|
+
end
|
633
|
+
end
|
634
|
+
|
635
|
+
describe "fill next year if january events arrive in december" do
|
636
|
+
config <<-CONFIG
|
637
|
+
filter {
|
638
|
+
date {
|
639
|
+
match => [ "message", "MMM dd HH:mm:ss" ]
|
640
|
+
timezone => "UTC"
|
641
|
+
}
|
642
|
+
}
|
643
|
+
CONFIG
|
644
|
+
|
645
|
+
before(:each) do
|
646
|
+
logstash_time = Time.utc(2013,12,31,23,59,50)
|
647
|
+
allow(Time).to receive(:now).and_return(logstash_time)
|
648
|
+
end
|
649
|
+
|
650
|
+
sample "Jan 01 01:00:00" do
|
651
|
+
insist { subject["@timestamp"].year } == 2014
|
652
|
+
end
|
653
|
+
end
|
654
|
+
end
|
529
655
|
end
|
metadata
CHANGED
@@ -1,91 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-date
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '>='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 2.0.0.beta2
|
20
|
-
- - <
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 3.0.0
|
23
14
|
requirement: !ruby/object:Gem::Requirement
|
24
15
|
requirements:
|
25
|
-
- -
|
16
|
+
- - "~>"
|
26
17
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
28
|
-
|
29
|
-
- !ruby/object:Gem::Version
|
30
|
-
version: 3.0.0
|
18
|
+
version: '1.0'
|
19
|
+
name: logstash-core-plugin-api
|
31
20
|
prerelease: false
|
32
21
|
type: :runtime
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-input-generator
|
35
22
|
version_requirements: !ruby/object:Gem::Requirement
|
36
23
|
requirements:
|
37
|
-
- -
|
24
|
+
- - "~>"
|
38
25
|
- !ruby/object:Gem::Version
|
39
|
-
version: '0'
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
40
28
|
requirement: !ruby/object:Gem::Requirement
|
41
29
|
requirements:
|
42
|
-
- -
|
30
|
+
- - ">="
|
43
31
|
- !ruby/object:Gem::Version
|
44
32
|
version: '0'
|
33
|
+
name: logstash-input-generator
|
45
34
|
prerelease: false
|
46
35
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: logstash-codec-json
|
49
36
|
version_requirements: !ruby/object:Gem::Requirement
|
50
37
|
requirements:
|
51
|
-
- -
|
38
|
+
- - ">="
|
52
39
|
- !ruby/object:Gem::Version
|
53
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
54
42
|
requirement: !ruby/object:Gem::Requirement
|
55
43
|
requirements:
|
56
|
-
- -
|
44
|
+
- - ">="
|
57
45
|
- !ruby/object:Gem::Version
|
58
46
|
version: '0'
|
47
|
+
name: logstash-codec-json
|
59
48
|
prerelease: false
|
60
49
|
type: :runtime
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
|
-
name: logstash-output-null
|
63
50
|
version_requirements: !ruby/object:Gem::Requirement
|
64
51
|
requirements:
|
65
|
-
- -
|
52
|
+
- - ">="
|
66
53
|
- !ruby/object:Gem::Version
|
67
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
68
56
|
requirement: !ruby/object:Gem::Requirement
|
69
57
|
requirements:
|
70
|
-
- -
|
58
|
+
- - ">="
|
71
59
|
- !ruby/object:Gem::Version
|
72
60
|
version: '0'
|
61
|
+
name: logstash-output-null
|
73
62
|
prerelease: false
|
74
63
|
type: :runtime
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
name: logstash-devutils
|
77
64
|
version_requirements: !ruby/object:Gem::Requirement
|
78
65
|
requirements:
|
79
|
-
- -
|
66
|
+
- - ">="
|
80
67
|
- !ruby/object:Gem::Version
|
81
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
82
70
|
requirement: !ruby/object:Gem::Requirement
|
83
71
|
requirements:
|
84
|
-
- -
|
72
|
+
- - ">="
|
85
73
|
- !ruby/object:Gem::Version
|
86
74
|
version: '0'
|
75
|
+
name: logstash-devutils
|
87
76
|
prerelease: false
|
88
77
|
type: :development
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
89
83
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
90
84
|
email: info@elastic.co
|
91
85
|
executables: []
|
@@ -113,12 +107,12 @@ require_paths:
|
|
113
107
|
- lib
|
114
108
|
required_ruby_version: !ruby/object:Gem::Requirement
|
115
109
|
requirements:
|
116
|
-
- -
|
110
|
+
- - ">="
|
117
111
|
- !ruby/object:Gem::Version
|
118
112
|
version: '0'
|
119
113
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
114
|
requirements:
|
121
|
-
- -
|
115
|
+
- - ">="
|
122
116
|
- !ruby/object:Gem::Version
|
123
117
|
version: '0'
|
124
118
|
requirements: []
|