fluentd 1.9.0-x64-mingw32 → 1.9.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.github/PULL_REQUEST_TEMPLATE.md +2 -1
  3. data/CHANGELOG.md +24 -0
  4. data/Gemfile +0 -2
  5. data/appveyor.yml +5 -14
  6. data/fluentd.gemspec +2 -1
  7. data/lib/fluent/config/section.rb +4 -0
  8. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  9. data/lib/fluent/plugin/in_tail.rb +12 -139
  10. data/lib/fluent/plugin/in_tail/position_file.rb +171 -0
  11. data/lib/fluent/plugin/out_forward.rb +3 -2
  12. data/lib/fluent/plugin/out_http.rb +10 -4
  13. data/lib/fluent/plugin/output.rb +1 -1
  14. data/lib/fluent/plugin/parser_syslog.rb +5 -2
  15. data/lib/fluent/plugin_helper/cert_option.rb +5 -2
  16. data/lib/fluent/plugin_helper/http_server.rb +62 -2
  17. data/lib/fluent/plugin_helper/http_server/compat/server.rb +14 -3
  18. data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +52 -0
  19. data/lib/fluent/plugin_helper/http_server/server.rb +14 -8
  20. data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb +41 -0
  21. data/lib/fluent/plugin_helper/server.rb +5 -10
  22. data/lib/fluent/plugin_helper/socket.rb +4 -8
  23. data/lib/fluent/tls.rb +81 -0
  24. data/lib/fluent/version.rb +1 -1
  25. data/test/config/test_section.rb +0 -2
  26. data/test/plugin/in_tail/test_position_file.rb +192 -0
  27. data/test/plugin/test_in_tail.rb +13 -0
  28. data/test/plugin/test_out_http.rb +15 -2
  29. data/test/plugin/test_output_as_buffered_backup.rb +2 -1
  30. data/test/plugin/test_parser_syslog.rb +36 -0
  31. data/test/plugin_helper/data/cert/generate_cert.rb +87 -0
  32. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +30 -0
  33. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +27 -0
  34. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +20 -0
  35. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +20 -0
  36. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +30 -0
  37. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +27 -0
  38. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +21 -0
  39. data/test/plugin_helper/data/cert/with_ca/cert.pem +21 -0
  40. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +30 -0
  41. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +27 -0
  42. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +20 -0
  43. data/test/plugin_helper/data/cert/without_ca/cert.pem +20 -0
  44. data/test/plugin_helper/test_http_server_helper.rb +168 -7
  45. data/test/plugin_helper/test_server.rb +40 -9
  46. data/test/test_tls.rb +65 -0
  47. metadata +52 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2d2a97ef2163328aefd7d64e066a07c753489aff4a3552f51b2cee964e5ac17c
4
- data.tar.gz: 7d1f18e584bcf15867d05235417389d52bfb22509f878b4fd934087491802ada
3
+ metadata.gz: 6f21baedc030fb756b14e72fabb65059322c86f07537c3328175eb982061ce80
4
+ data.tar.gz: b4a596b4cf6704dd752cff7d19af6618b3ab40c627d052a067174514b21157be
5
5
  SHA512:
6
- metadata.gz: d612a1e30b3e61a2ef0c1022e4aefcf41dd661f4d7dfef05e01a5ce6b54a36df866306d505b24b4e051bb50e7d2b3b506d4f8426df280c6032d9281eb1fe0a25
7
- data.tar.gz: d638c3d0aa0d0bde6fdb1603ebbbb9d502ad66c203d5f3162de37db665670039c7b8b60984f709462ca4aef2d9332446c37e618eeeb205b0e0c9293a6456afd9
6
+ metadata.gz: ce0465ee776348e12636d3b003f9a54a2da7c1d643117bae037af882c4511cfe036c0145c314dcfa542ed47715c891db268acd35dab9c99053f6ebc8d32a476e
7
+ data.tar.gz: '009d91ce4bd655b67e51fc93f7df5de59d82fc880bf9b5930dbcd904ada392d35aa833abf08a8211a35289994871d733ea6d5862232884abe6676a23b83ff4f4'
@@ -1,6 +1,7 @@
1
1
  <!--
2
2
  Thank you for contributing to Fluentd!
3
- Please provide the following information to help us make the most of your pull request:
3
+ Your commits need to follow DCO: https://probot.github.io/apps/dco/
4
+ And please provide the following information to help us make the most of your pull request:
4
5
  -->
5
6
 
6
7
  **Which issue(s) this PR fixes**:
data/CHANGELOG.md CHANGED
@@ -1,5 +1,29 @@
1
1
  # v1.9
2
2
 
3
+ ## Release v1.9.1 - 2020/01/31
4
+
5
+ ### Enhancement
6
+
7
+ * http_server helper: Support HTTPS
8
+ https://github.com/fluent/fluentd/pull/2787
9
+ * in_tail: Add `path_delimiter` to split with any char
10
+ https://github.com/fluent/fluentd/pull/2796
11
+ * in_tail: Remove an entry from PositionaFile when it is unwatched
12
+ https://github.com/fluent/fluentd/pull/2803
13
+ * out_http: Add warning for `retryable_response_code`
14
+ https://github.com/fluent/fluentd/pull/2809
15
+ * parser_syslog: Add multiline RFC5424 support
16
+ https://github.com/fluent/fluentd/pull/2767
17
+ * Add TLS module to unify TLS related code
18
+ https://github.com/fluent/fluentd/pull/2802
19
+
20
+ ### Bug fix
21
+
22
+ * output: Add `EncodingError` to unrecoverable errors
23
+ https://github.com/fluent/fluentd/pull/2808
24
+ * tls: Fix TLS version handling in secure mode
25
+ https://github.com/fluent/fluentd/pull/2802
26
+
3
27
  ## Release v1.9.0 - 2020/01/22
4
28
 
5
29
  ### New feature
data/Gemfile CHANGED
@@ -2,8 +2,6 @@ source 'https://rubygems.org/'
2
2
 
3
3
  gemspec
4
4
 
5
- gem 'async-http', '~> 0.42'
6
-
7
5
  local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
8
6
  if File.exist?(local_gemfile)
9
7
  puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v`
data/appveyor.yml CHANGED
@@ -5,11 +5,9 @@ version: '{build}'
5
5
 
6
6
  install:
7
7
  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
8
- - IF %ridk%==0 "%devkit%\\devkitvars.bat"
9
8
  - ruby --version
10
9
  - gem --version
11
- - IF %ridk%==1 ridk.cmd exec bundle install
12
- - IF %ridk%==0 bundle install
10
+ - ridk.cmd exec bundle install
13
11
  build: off
14
12
  test_script:
15
13
  - bundle exec rake test
@@ -22,16 +20,9 @@ branches:
22
20
  # https://www.appveyor.com/docs/installed-software/#ruby
23
21
  environment:
24
22
  matrix:
25
- - ruby_version: "23-x64"
26
- devkit: C:\Ruby23-x64\DevKit
27
- ridk: 0
28
- - ruby_version: "23"
29
- devkit: C:\Ruby23\DevKit
30
- ridk: 0
23
+ - ruby_version: "26-x64"
24
+ - ruby_version: "26"
25
+ - ruby_version: "25-x64"
26
+ - ruby_version: "25"
31
27
  - ruby_version: "24-x64"
32
- ridk: 1
33
28
  - ruby_version: "24"
34
- ridk: 1
35
- matrix:
36
- allow_failures:
37
- - ruby_version: "21"
data/fluentd.gemspec CHANGED
@@ -48,5 +48,6 @@ Gem::Specification.new do |gem|
48
48
  gem.add_development_dependency("test-unit", ["~> 3.3"])
49
49
  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
50
50
  gem.add_development_dependency("oj", [">= 2.14", "< 4"])
51
- gem.add_development_dependency("ext_monitor", [">= 0.1.1", "< 0.2"])
51
+ gem.add_development_dependency("ext_monitor", [">= 0.1.2", "< 0.2"])
52
+ gem.add_development_dependency("async-http")
52
53
  end
@@ -58,6 +58,10 @@ module Fluent
58
58
  @params
59
59
  end
60
60
 
61
+ def dup
62
+ Section.new(@params.dup, @corresponding_config_element.dup)
63
+ end
64
+
61
65
  def +(other)
62
66
  Section.new(self.to_h.merge(other.to_h))
63
67
  end
@@ -210,7 +210,7 @@ module Fluent::Plugin
210
210
 
211
211
  log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
212
212
  api_handler = APIHandler.new(self)
213
- create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
213
+ http_server_create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
214
214
  serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }
215
215
  serv.get('/api/plugins.json') { |req| api_handler.plugins_json(req) }
216
216
  serv.get('/api/config') { |req| api_handler.config_ltsv(req) }
@@ -22,6 +22,7 @@ require 'fluent/event'
22
22
  require 'fluent/plugin/buffer'
23
23
  require 'fluent/plugin/parser_multiline'
24
24
  require 'fluent/variable_store'
25
+ require 'fluent/plugin/in_tail/position_file'
25
26
 
26
27
  if Fluent.windows?
27
28
  require_relative 'file_wrapper'
@@ -35,6 +36,8 @@ module Fluent::Plugin
35
36
 
36
37
  helpers :timer, :event_loop, :parser, :compat_parameters
37
38
 
39
+ RESERVED_CHARS = ['/', '*', '%'].freeze
40
+
38
41
  class WatcherSetupError < StandardError
39
42
  def initialize(msg)
40
43
  @message = msg
@@ -58,6 +61,8 @@ module Fluent::Plugin
58
61
 
59
62
  desc 'The paths to read. Multiple paths can be specified, separated by comma.'
60
63
  config_param :path, :string
64
+ desc 'path delimiter used for spliting path config'
65
+ config_param :path_delimiter, :string, default: ','
61
66
  desc 'The tag of the event.'
62
67
  config_param :tag, :string
63
68
  desc 'The paths to exclude the files from watcher list.'
@@ -113,9 +118,6 @@ module Fluent::Plugin
113
118
  unless parser_config
114
119
  raise Fluent::ConfigError, "<parse> section is required."
115
120
  end
116
- unless parser_config["@type"]
117
- raise Fluent::ConfigError, "parse/@type is required."
118
- end
119
121
 
120
122
  (1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|
121
123
  parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
@@ -127,7 +129,12 @@ module Fluent::Plugin
127
129
  raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
128
130
  end
129
131
 
130
- @paths = @path.split(',').map {|path| path.strip }
132
+ if RESERVED_CHARS.include?(@path_delimiter)
133
+ rc = RESERVED_CHARS.join(', ')
134
+ raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
135
+ end
136
+
137
+ @paths = @path.split(@path_delimiter).map(&:strip)
131
138
  if @paths.empty?
132
139
  raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
133
140
  end
@@ -391,7 +398,7 @@ module Fluent::Plugin
391
398
  tw.close if close_io
392
399
  flush_buffer(tw)
393
400
  if tw.unwatched && @pf
394
- @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
401
+ @pf.unwatch(tw.path)
395
402
  end
396
403
  end
397
404
 
@@ -913,139 +920,5 @@ module Fluent::Plugin
913
920
  end
914
921
  end
915
922
  end
916
-
917
- class PositionFile
918
- UNWATCHED_POSITION = 0xffffffffffffffff
919
-
920
- def initialize(file, file_mutex, map, last_pos)
921
- @file = file
922
- @file_mutex = file_mutex
923
- @map = map
924
- @last_pos = last_pos
925
- end
926
-
927
- def [](path)
928
- if m = @map[path]
929
- return m
930
- end
931
-
932
- @file_mutex.synchronize {
933
- @file.pos = @last_pos
934
- @file.write "#{path}\t0000000000000000\t0000000000000000\n"
935
- seek = @last_pos + path.bytesize + 1
936
- @last_pos = @file.pos
937
- @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
938
- }
939
- end
940
-
941
- def self.parse(file)
942
- compact(file)
943
-
944
- file_mutex = Mutex.new
945
- map = {}
946
- file.pos = 0
947
- file.each_line {|line|
948
- m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
949
- unless m
950
- $log.warn "Unparsable line in pos_file: #{line}"
951
- next
952
- end
953
- path = m[1]
954
- pos = m[2].to_i(16)
955
- ino = m[3].to_i(16)
956
- seek = file.pos - line.bytesize + path.bytesize + 1
957
- map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
958
- }
959
- new(file, file_mutex, map, file.pos)
960
- end
961
-
962
- # Clean up unwatched file entries
963
- def self.compact(file)
964
- file.pos = 0
965
- existent_entries = file.each_line.map { |line|
966
- m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
967
- unless m
968
- $log.warn "Unparsable line in pos_file: #{line}"
969
- next
970
- end
971
- path = m[1]
972
- pos = m[2].to_i(16)
973
- ino = m[3].to_i(16)
974
- # 32bit inode converted to 64bit at this phase
975
- pos == UNWATCHED_POSITION ? nil : ("%s\t%016x\t%016x\n" % [path, pos, ino])
976
- }.compact
977
-
978
- file.pos = 0
979
- file.truncate(0)
980
- file.write(existent_entries.join)
981
- end
982
- end
983
-
984
- # pos inode
985
- # ffffffffffffffff\tffffffffffffffff\n
986
- class FilePositionEntry
987
- POS_SIZE = 16
988
- INO_OFFSET = 17
989
- INO_SIZE = 16
990
- LN_OFFSET = 33
991
- SIZE = 34
992
-
993
- def initialize(file, file_mutex, seek, pos, inode)
994
- @file = file
995
- @file_mutex = file_mutex
996
- @seek = seek
997
- @pos = pos
998
- @inode = inode
999
- end
1000
-
1001
- def update(ino, pos)
1002
- @file_mutex.synchronize {
1003
- @file.pos = @seek
1004
- @file.write "%016x\t%016x" % [pos, ino]
1005
- }
1006
- @pos = pos
1007
- @inode = ino
1008
- end
1009
-
1010
- def update_pos(pos)
1011
- @file_mutex.synchronize {
1012
- @file.pos = @seek
1013
- @file.write "%016x" % pos
1014
- }
1015
- @pos = pos
1016
- end
1017
-
1018
- def read_inode
1019
- @inode
1020
- end
1021
-
1022
- def read_pos
1023
- @pos
1024
- end
1025
- end
1026
-
1027
- class MemoryPositionEntry
1028
- def initialize
1029
- @pos = 0
1030
- @inode = 0
1031
- end
1032
-
1033
- def update(ino, pos)
1034
- @inode = ino
1035
- @pos = pos
1036
- end
1037
-
1038
- def update_pos(pos)
1039
- @pos = pos
1040
- end
1041
-
1042
- def read_pos
1043
- @pos
1044
- end
1045
-
1046
- def read_inode
1047
- @inode
1048
- end
1049
- end
1050
923
  end
1051
924
  end
@@ -0,0 +1,171 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/in_tail'
18
+
19
+ module Fluent::Plugin
20
+ class TailInput < Fluent::Plugin::Input
21
+ class PositionFile
22
+ UNWATCHED_POSITION = 0xffffffffffffffff
23
+ POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
24
+ POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
25
+
26
+ def initialize(file, file_mutex, map)
27
+ @file = file
28
+ @file_mutex = file_mutex
29
+ @map = map
30
+ end
31
+
32
+ def [](path)
33
+ if m = @map[path]
34
+ return m
35
+ end
36
+
37
+ @file_mutex.synchronize {
38
+ @file.seek(0, IO::SEEK_END)
39
+ seek = @file.pos + path.bytesize + 1
40
+ @file.write "#{path}\t0000000000000000\t0000000000000000\n"
41
+ @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
42
+ }
43
+ end
44
+
45
+ def unwatch(path)
46
+ if (entry = @map.delete(path))
47
+ entry.update_pos(UNWATCHED_POSITION)
48
+ end
49
+ end
50
+
51
+ def self.parse(file)
52
+ compact(file)
53
+
54
+ file_mutex = Mutex.new
55
+ map = {}
56
+ file.pos = 0
57
+ file.each_line {|line|
58
+ m = POSITION_FILE_ENTRY_REGEX.match(line)
59
+ unless m
60
+ $log.warn "Unparsable line in pos_file: #{line}"
61
+ next
62
+ end
63
+ path = m[1]
64
+ pos = m[2].to_i(16)
65
+ ino = m[3].to_i(16)
66
+ seek = file.pos - line.bytesize + path.bytesize + 1
67
+ map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
68
+ }
69
+ new(file, file_mutex, map)
70
+ end
71
+
72
+ # Clean up unwatched file entries
73
+ def self.compact(file)
74
+ existent_entries = {}
75
+ file.pos = 0
76
+ file.each_line do |line|
77
+ m = POSITION_FILE_ENTRY_REGEX.match(line)
78
+ unless m
79
+ $log.warn "Unparsable line in pos_file: #{line}"
80
+ next
81
+ end
82
+ path = m[1]
83
+ pos = m[2].to_i(16)
84
+ ino = m[3].to_i(16)
85
+
86
+ if pos == UNWATCHED_POSITION
87
+ next
88
+ end
89
+
90
+ if existent_entries.include?(path)
91
+ $log.warn("#{path} already exists. use latest one: deleted #{existent_entries[path]}")
92
+ end
93
+
94
+ # 32bit inode converted to 64bit at this phase
95
+ existent_entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
96
+ end
97
+
98
+ file.pos = 0
99
+ file.truncate(0)
100
+ file.write(existent_entries.values.join)
101
+ end
102
+ end
103
+
104
+ # pos inode
105
+ # ffffffffffffffff\tffffffffffffffff\n
106
+ class FilePositionEntry
107
+ POS_SIZE = 16
108
+ INO_OFFSET = 17
109
+ INO_SIZE = 16
110
+ LN_OFFSET = 33
111
+ SIZE = 34
112
+
113
+ def initialize(file, file_mutex, seek, pos, inode)
114
+ @file = file
115
+ @file_mutex = file_mutex
116
+ @seek = seek
117
+ @pos = pos
118
+ @inode = inode
119
+ end
120
+
121
+ def update(ino, pos)
122
+ @file_mutex.synchronize {
123
+ @file.pos = @seek
124
+ @file.write "%016x\t%016x" % [pos, ino]
125
+ }
126
+ @pos = pos
127
+ @inode = ino
128
+ end
129
+
130
+ def update_pos(pos)
131
+ @file_mutex.synchronize {
132
+ @file.pos = @seek
133
+ @file.write "%016x" % pos
134
+ }
135
+ @pos = pos
136
+ end
137
+
138
+ def read_inode
139
+ @inode
140
+ end
141
+
142
+ def read_pos
143
+ @pos
144
+ end
145
+ end
146
+
147
+ class MemoryPositionEntry
148
+ def initialize
149
+ @pos = 0
150
+ @inode = 0
151
+ end
152
+
153
+ def update(ino, pos)
154
+ @inode = ino
155
+ @pos = pos
156
+ end
157
+
158
+ def update_pos(pos)
159
+ @pos = pos
160
+ end
161
+
162
+ def read_pos
163
+ @pos
164
+ end
165
+
166
+ def read_inode
167
+ @inode
168
+ end
169
+ end
170
+ end
171
+ end