logstash-filter-date 2.1.2 → 2.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: eb61244507b109cd3259147af0cfd04a15747f7c
4
- data.tar.gz: 2a79943e40e56365ad2a82b40f3b30c978312072
3
+ metadata.gz: ea6c51877b6a28e814bd00e623932612e5b9f8f1
4
+ data.tar.gz: ebe269a2eb0210e9b4eedcb796ecedc06bb830e4
5
5
  SHA512:
6
- metadata.gz: 095c0061ca96d90605ce2c9ca8c6a19a62858ab0149ea2fa21adfc1d1bbbf4e36f5a9b3ca8a1d84449e1f7b04cda1afe140226c420a364d4fd7dd506071936ae
7
- data.tar.gz: 6faabecb214c5f18290cb6ae7ad1141c57bb7a5229cb193771d0f028980cc4c18099c338cc85b6d27f356b6eaa9c3f0a1016df915819cec4104af702cd009633
6
+ metadata.gz: 8d905fb89de9f982b6b2125bab4adfc68861d17dcfd4d0bb92b3497382681964f29c82e7e26ee4930fb83aaa13929dcc8a54daac7c5063f430eca182337138ad
7
+ data.tar.gz: 75b71d3e79ac533ab6ad51d4b2ee749847d841f0601a9f8db44577b70941e4df30c534df9996e7ec6b220d5c556e94e11eb3457c001017ec4edef7cc32c232db
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ # 2.1.4
2
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
3
+ # 2.1.3
4
+ - New dependency requirements for logstash-core for the 5.0 release
1
5
  ## 2.1.2
2
6
  - Make tests less reliant on implementation details of LogStash::Event
3
7
  ## 2.1.1
@@ -48,11 +48,6 @@ class LogStash::Filters::Date < LogStash::Filters::Base
48
48
  # an english parser will also be used as a fallback mechanism.
49
49
  config :locale, :validate => :string
50
50
 
51
- # The date formats allowed are anything allowed by Joda-Time (java time
52
- # library). You can see the docs for this format here:
53
- #
54
- # http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html[joda.time.format.DateTimeFormat]
55
- #
56
51
  # An array with field name first, and format patterns following, `[ field,
57
52
  # formats... ]`
58
53
  #
@@ -84,6 +79,71 @@ class LogStash::Filters::Date < LogStash::Filters::Base
84
79
  # If your field is nested in your structure, you can use the nested
85
80
  # syntax `[foo][bar]` to match its value. For more information, please refer to
86
81
  # <<logstash-config-field-references>>
82
+ #
83
+ # *More details on the syntax*
84
+ #
85
+ # The syntax used for parsing date and time text uses letters to indicate the
86
+ # kind of time value (month, minute, etc), and a repetition of letters to
87
+ # indicate the form of that value (2-digit month, full month name, etc).
88
+ #
89
+ # Here's what you can use to parse dates and times:
90
+ #
91
+ # [horizontal]
92
+ # y:: year
93
+ # yyyy::: full year number. Example: `2015`.
94
+ # yy::: two-digit year. Example: `15` for the year 2015.
95
+ #
96
+ # M:: month of the year
97
+ # M::: minimal-digit month. Example: `1` for January and `12` for December.
98
+ # MM::: two-digit month. zero-padded if needed. Example: `01` for January and `12` for December
99
+ # MMM::: abbreviated month text. Example: `Jan` for January. Note: The language used depends on your locale. See the `locale` setting for how to change the language.
100
+ # MMMM::: full month text, Example: `January`. Note: The language used depends on your locale.
101
+ #
102
+ # d:: day of the month
103
+ # d::: minimal-digit day. Example: `1` for the 1st of the month.
104
+ # dd::: two-digit day, zero-padded if needed. Example: `01` for the 1st of the month.
105
+ #
106
+ # H:: hour of the day (24-hour clock)
107
+ # H::: minimal-digit hour. Example: `0` for midnight.
108
+ # HH::: two-digit hour, zero-padded if needed. Example: `00` for midnight.
109
+ #
110
+ # m:: minutes of the hour (60 minutes per hour)
111
+ # m::: minimal-digit minutes. Example: `0`.
112
+ # mm::: two-digit minutes, zero-padded if needed. Example: `00`.
113
+ #
114
+ # s:: seconds of the minute (60 seconds per minute)
115
+ # s::: minimal-digit seconds. Example: `0`.
116
+ # ss::: two-digit seconds, zero-padded if needed. Example: `00`.
117
+ #
118
+ # S:: fraction of a second
119
+ # *Maximum precision is milliseconds (`SSS`). Beyond that, zeroes are appended.*
120
+ # S::: tenths of a second. Example: `0` for a subsecond value `012`
121
+ # SS::: hundredths of a second. Example: `01` for a subsecond value `01`
122
+ # SSS::: thousandths of a second. Example: `012` for a subsecond value `012`
123
+ #
124
+ # Z:: time zone offset or identity
125
+ # Z::: Timezone offset structured as HHmm (hour and minutes offset from Zulu/UTC). Example: `-0700`.
126
+ # ZZ::: Timezone offset structured as HH:mm (colon in between hour and minute offsets). Example: `-07:00`.
127
+ # ZZZ::: Timezone identity. Example: `America/Los_Angeles`. Note: Valid IDs are listed on the http://joda-time.sourceforge.net/timezones.html[Joda.org available time zones page].
128
+ #
129
+ # z:: time zone names. *Time zone names ('z') cannot be parsed.*
130
+ #
131
+ # w:: week of the year
132
+ # w::: minimal-digit week. Example: `1`.
133
+ # ww::: two-digit week, zero-padded if needed. Example: `01`.
134
+ #
135
+ # D:: day of the year
136
+ #
137
+ # e:: day of the week (number)
138
+ #
139
+ # E:: day of the week (text)
140
+ # E, EE, EEE::: Abbreviated day of the week. Example: `Mon`, `Tue`, `Wed`, `Thu`, `Fri`, `Sat`, `Sun`. Note: The actual language of this will depend on your locale.
141
+ # EEEE::: The full text day of the week. Example: `Monday`, `Tuesday`, ... Note: The actual language of this will depend on your locale.
142
+ #
143
+ # For non-formatting syntax, you'll need to put single-quote characters around the value. For example, if you were parsing ISO8601 time, "2015-01-01T01:12:23" that little "T" isn't a valid time format, and you want to say "literally, a T", your format would be this: "yyyy-MM-dd'T'HH:mm:ss"
144
+ #
145
+ # Other less common date units, such as era (G), century \(C), am/pm (a), and # more, can be learned about on the
146
+ # http://www.joda.org/joda-time/key_format.html[joda-time documentation].
87
147
  config :match, :validate => :array, :default => []
88
148
 
89
149
  # Store the matching timestamp into the given target field. If not provided,
@@ -125,6 +185,24 @@ class LogStash::Filters::Date < LogStash::Filters::Base
125
185
  setupMatcher(@config["match"].shift, locale, @config["match"] )
126
186
  end
127
187
 
188
+ def parseWithJodaParser(joda_parser, date, format_has_year)
189
+ return joda_parser.parseMillis(date) if format_has_year
190
+ now = Time.now
191
+ now_month = now.month
192
+ result = joda_parser.parseDateTime(date)
193
+ event_month = result.getMonthOfYear
194
+
195
+ if (event_month == now_month)
196
+ result.with_year(now.year)
197
+ elsif (event_month == 12 && now_month == 1)
198
+ result.with_year(now.year-1)
199
+ elsif (event_month == 1 && now_month == 12)
200
+ result.with_year(now.year+1)
201
+ else
202
+ result.with_year(now.year)
203
+ end.get_millis
204
+ end
205
+
128
206
  def setupMatcher(field, locale, value)
129
207
  value.each do |format|
130
208
  parsers = []
@@ -181,25 +259,11 @@ class LogStash::Filters::Date < LogStash::Filters::Base
181
259
  end
182
260
  if @sprintf_timezone
183
261
  parsers << lambda { |date , tz|
184
- joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)).parseMillis(date)
262
+ return parseWithJodaParser(joda_parser.withZone(org.joda.time.DateTimeZone.forID(tz)), date, format_has_year)
185
263
  }
186
264
  end
187
265
  parsers << lambda do |date|
188
- return joda_parser.parseMillis(date) if format_has_year
189
- now = Time.now
190
- now_month = now.month
191
- result = joda_parser.parseDateTime(date)
192
- event_month = result.getMonthOfYear
193
-
194
- if (event_month == now_month)
195
- result.with_year(now.year)
196
- elsif (event_month == 12 && now_month == 1)
197
- result.with_year(now.year-1)
198
- elsif (event_month == 1 && now_month == 12)
199
- result.with_year(now.year+1)
200
- else
201
- result.with_year(now.year)
202
- end.get_millis
266
+ return parseWithJodaParser(joda_parser, date, format_has_year)
203
267
  end
204
268
 
205
269
  #Include a fallback parser to english when default locale is non-english
@@ -207,7 +271,7 @@ class LogStash::Filters::Date < LogStash::Filters::Base
207
271
  "en" != java.util.Locale.getDefault().getLanguage() &&
208
272
  (format.include?("MMM") || format.include?("E"))
209
273
  en_joda_parser = joda_parser.withLocale(java.util.Locale.forLanguageTag('en-US'))
210
- parsers << lambda { |date| en_joda_parser.parseMillis(date) }
274
+ parsers << lambda { |date| parseWithJodaParser(en_joda_parser, date, format_has_year) }
211
275
  end
212
276
  rescue JavaException => e
213
277
  raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-date'
4
- s.version = '2.1.2'
4
+ s.version = '2.1.4'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
20
20
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
21
21
 
22
22
  # Gem dependencies
23
- s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
23
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
24
24
  s.add_runtime_dependency 'logstash-input-generator'
25
25
  s.add_runtime_dependency 'logstash-codec-json'
26
26
  s.add_runtime_dependency 'logstash-output-null'
@@ -386,6 +386,67 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
386
386
  end # times.each
387
387
  end
388
388
 
389
+ context "Default year handling when parsing with timezone from event" do
390
+
391
+ describe "LOGSTASH-34 - Default year should be this year" do
392
+ config <<-CONFIG
393
+ filter {
394
+ date {
395
+ match => [ "message", "EEE MMM dd HH:mm:ss" ]
396
+ locale => "en"
397
+ timezone => "%{mytz}"
398
+ }
399
+ }
400
+ CONFIG
401
+
402
+ sample("message" => "Sun Jun 02 20:38:03", "mytz" => "UTC") do
403
+ insist { subject["@timestamp"].year } == Time.now.year
404
+ end
405
+ end
406
+
407
+ describe "fill last year if december events arrive in january" do
408
+ config <<-CONFIG
409
+ filter {
410
+ date {
411
+ match => [ "message", "MMM dd HH:mm:ss" ]
412
+ locale => "en"
413
+ timezone => "%{mytz}"
414
+ }
415
+ }
416
+ CONFIG
417
+
418
+ before(:each) do
419
+ logstash_time = Time.utc(2014,1,1,00,30,50)
420
+ allow(Time).to receive(:now).and_return(logstash_time)
421
+ end
422
+
423
+ sample("message" => "Dec 31 23:59:00", "mytz" => "UTC") do
424
+ insist { subject["@timestamp"].year } == 2013
425
+ end
426
+ end
427
+
428
+ describe "fill next year if january events arrive in december" do
429
+ config <<-CONFIG
430
+ filter {
431
+ date {
432
+ match => [ "message", "MMM dd HH:mm:ss" ]
433
+ locale => "en"
434
+ timezone => "%{mytz}"
435
+ }
436
+ }
437
+ CONFIG
438
+
439
+ before(:each) do
440
+ logstash_time = Time.utc(2013,12,31,23,59,50)
441
+ allow(Time).to receive(:now).and_return(logstash_time)
442
+ end
443
+
444
+ sample( "message" => "Jan 01 01:00:00", "mytz" => "UTC") do
445
+ insist { subject["@timestamp"].year } == 2014
446
+ end
447
+ end
448
+ end
449
+
389
450
  describe "LOGSTASH-34 - Default year should be this year" do
390
451
  config <<-CONFIG
391
452
  filter {
@@ -526,4 +587,69 @@ RUBY_ENGINE == "jruby" and describe LogStash::Filters::Date do
526
587
  #Restore default locale
527
588
  java.util.Locale.setDefault(default_locale)
528
589
  end
590
+
591
+ context "Default year handling when parsing with english fallback parser" do
592
+
593
+ before(:each) do
594
+ default_locale = java.util.Locale.getDefault()
595
+ #Override default locale with non-english
596
+ java.util.Locale.setDefault(java.util.Locale.forLanguageTag('fr-FR'))
597
+ end
598
+
599
+ puts "override locale"
600
+ describe "LOGSTASH-34 - Default year should be this year" do
601
+ config <<-CONFIG
602
+ filter {
603
+ date {
604
+ match => [ "message", "EEE MMM dd HH:mm:ss" ]
605
+ timezone => "UTC"
606
+ }
607
+ }
608
+ CONFIG
609
+
610
+ sample "Sun Jun 02 20:38:03" do
611
+ insist { subject["@timestamp"].year } == Time.now.year
612
+ end
613
+ end
614
+
615
+ describe "fill last year if december events arrive in january" do
616
+ config <<-CONFIG
617
+ filter {
618
+ date {
619
+ match => [ "message", "MMM dd HH:mm:ss" ]
620
+ timezone => "UTC"
621
+ }
622
+ }
623
+ CONFIG
624
+
625
+ before(:each) do
626
+ logstash_time = Time.utc(2014,1,1,00,30,50)
627
+ allow(Time).to receive(:now).and_return(logstash_time)
628
+ end
629
+
630
+ sample "Dec 31 23:59:00" do
631
+ insist { subject["@timestamp"].year } == 2013
632
+ end
633
+ end
634
+
635
+ describe "fill next year if january events arrive in december" do
636
+ config <<-CONFIG
637
+ filter {
638
+ date {
639
+ match => [ "message", "MMM dd HH:mm:ss" ]
640
+ timezone => "UTC"
641
+ }
642
+ }
643
+ CONFIG
644
+
645
+ before(:each) do
646
+ logstash_time = Time.utc(2013,12,31,23,59,50)
647
+ allow(Time).to receive(:now).and_return(logstash_time)
648
+ end
649
+
650
+ sample "Jan 01 01:00:00" do
651
+ insist { subject["@timestamp"].year } == 2014
652
+ end
653
+ end
654
+ end
529
655
  end
metadata CHANGED
@@ -1,91 +1,85 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-date
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.2
4
+ version: 2.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-09 00:00:00.000000000 Z
11
+ date: 2016-03-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: logstash-core
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '>='
18
- - !ruby/object:Gem::Version
19
- version: 2.0.0.beta2
20
- - - <
21
- - !ruby/object:Gem::Version
22
- version: 3.0.0
23
14
  requirement: !ruby/object:Gem::Requirement
24
15
  requirements:
25
- - - '>='
16
+ - - "~>"
26
17
  - !ruby/object:Gem::Version
27
- version: 2.0.0.beta2
28
- - - <
29
- - !ruby/object:Gem::Version
30
- version: 3.0.0
18
+ version: '1.0'
19
+ name: logstash-core-plugin-api
31
20
  prerelease: false
32
21
  type: :runtime
33
- - !ruby/object:Gem::Dependency
34
- name: logstash-input-generator
35
22
  version_requirements: !ruby/object:Gem::Requirement
36
23
  requirements:
37
- - - '>='
24
+ - - "~>"
38
25
  - !ruby/object:Gem::Version
39
- version: '0'
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
40
28
  requirement: !ruby/object:Gem::Requirement
41
29
  requirements:
42
- - - '>='
30
+ - - ">="
43
31
  - !ruby/object:Gem::Version
44
32
  version: '0'
33
+ name: logstash-input-generator
45
34
  prerelease: false
46
35
  type: :runtime
47
- - !ruby/object:Gem::Dependency
48
- name: logstash-codec-json
49
36
  version_requirements: !ruby/object:Gem::Requirement
50
37
  requirements:
51
- - - '>='
38
+ - - ">="
52
39
  - !ruby/object:Gem::Version
53
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
54
42
  requirement: !ruby/object:Gem::Requirement
55
43
  requirements:
56
- - - '>='
44
+ - - ">="
57
45
  - !ruby/object:Gem::Version
58
46
  version: '0'
47
+ name: logstash-codec-json
59
48
  prerelease: false
60
49
  type: :runtime
61
- - !ruby/object:Gem::Dependency
62
- name: logstash-output-null
63
50
  version_requirements: !ruby/object:Gem::Requirement
64
51
  requirements:
65
- - - '>='
52
+ - - ">="
66
53
  - !ruby/object:Gem::Version
67
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
68
56
  requirement: !ruby/object:Gem::Requirement
69
57
  requirements:
70
- - - '>='
58
+ - - ">="
71
59
  - !ruby/object:Gem::Version
72
60
  version: '0'
61
+ name: logstash-output-null
73
62
  prerelease: false
74
63
  type: :runtime
75
- - !ruby/object:Gem::Dependency
76
- name: logstash-devutils
77
64
  version_requirements: !ruby/object:Gem::Requirement
78
65
  requirements:
79
- - - '>='
66
+ - - ">="
80
67
  - !ruby/object:Gem::Version
81
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
82
70
  requirement: !ruby/object:Gem::Requirement
83
71
  requirements:
84
- - - '>='
72
+ - - ">="
85
73
  - !ruby/object:Gem::Version
86
74
  version: '0'
75
+ name: logstash-devutils
87
76
  prerelease: false
88
77
  type: :development
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
89
83
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
90
84
  email: info@elastic.co
91
85
  executables: []
@@ -113,12 +107,12 @@ require_paths:
113
107
  - lib
114
108
  required_ruby_version: !ruby/object:Gem::Requirement
115
109
  requirements:
116
- - - '>='
110
+ - - ">="
117
111
  - !ruby/object:Gem::Version
118
112
  version: '0'
119
113
  required_rubygems_version: !ruby/object:Gem::Requirement
120
114
  requirements:
121
- - - '>='
115
+ - - ">="
122
116
  - !ruby/object:Gem::Version
123
117
  version: '0'
124
118
  requirements: []