fluentd 1.14.4-x86-mingw32 → 1.15.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +9 -6
  6. data/CHANGELOG.md +115 -19
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +2 -23
  10. data/Rakefile +1 -1
  11. data/fluentd.gemspec +3 -1
  12. data/lib/fluent/command/ctl.rb +4 -1
  13. data/lib/fluent/command/fluentd.rb +14 -0
  14. data/lib/fluent/config/error.rb +12 -0
  15. data/lib/fluent/config/literal_parser.rb +2 -2
  16. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  17. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  18. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  19. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  20. data/lib/fluent/config/yaml_parser.rb +56 -0
  21. data/lib/fluent/config.rb +14 -1
  22. data/lib/fluent/event_router.rb +19 -1
  23. data/lib/fluent/plugin/bare_output.rb +1 -1
  24. data/lib/fluent/plugin/base.rb +1 -1
  25. data/lib/fluent/plugin/file_wrapper.rb +52 -107
  26. data/lib/fluent/plugin/in_forward.rb +1 -1
  27. data/lib/fluent/plugin/in_http.rb +11 -1
  28. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  29. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  30. data/lib/fluent/plugin/in_tail.rb +66 -47
  31. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  32. data/lib/fluent/plugin/output.rb +43 -33
  33. data/lib/fluent/plugin/parser.rb +3 -4
  34. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  35. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  36. data/lib/fluent/plugin_helper/server.rb +23 -4
  37. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  38. data/lib/fluent/plugin_helper/socket.rb +13 -2
  39. data/lib/fluent/registry.rb +2 -1
  40. data/lib/fluent/rpc.rb +4 -3
  41. data/lib/fluent/supervisor.rb +114 -27
  42. data/lib/fluent/system_config.rb +2 -1
  43. data/lib/fluent/version.rb +1 -1
  44. data/lib/fluent/winsvc.rb +2 -0
  45. data/test/command/test_ctl.rb +0 -1
  46. data/test/command/test_fluentd.rb +33 -0
  47. data/test/compat/test_parser.rb +1 -1
  48. data/test/config/test_system_config.rb +3 -1
  49. data/test/config/test_types.rb +1 -1
  50. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  51. data/test/plugin/in_tail/test_position_file.rb +0 -63
  52. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  53. data/test/plugin/test_file_wrapper.rb +0 -68
  54. data/test/plugin/test_filter_parser.rb +1 -1
  55. data/test/plugin/test_filter_stdout.rb +2 -2
  56. data/test/plugin/test_in_forward.rb +0 -2
  57. data/test/plugin/test_in_http.rb +23 -0
  58. data/test/plugin/test_in_object_space.rb +9 -3
  59. data/test/plugin/test_in_syslog.rb +1 -1
  60. data/test/plugin/test_in_tail.rb +629 -353
  61. data/test/plugin/test_out_forward.rb +30 -20
  62. data/test/plugin/test_out_stdout.rb +2 -2
  63. data/test/plugin/test_output_as_buffered_retries.rb +53 -6
  64. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  65. data/test/plugin/test_parser_syslog.rb +1 -1
  66. data/test/plugin_helper/test_cert_option.rb +1 -1
  67. data/test/plugin_helper/test_child_process.rb +16 -4
  68. data/test/plugin_helper/test_retry_state.rb +602 -38
  69. data/test/plugin_helper/test_server.rb +18 -0
  70. data/test/test_config.rb +135 -4
  71. data/test/test_event_router.rb +17 -0
  72. data/test/test_formatter.rb +1 -1
  73. data/test/test_supervisor.rb +196 -6
  74. metadata +39 -5
@@ -0,0 +1,166 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/yaml_parser/section_builder'
18
+
19
+ module Fluent
20
+ module Config
21
+ module YamlParser
22
+ class Parser
23
+ def initialize(config, indent: 2)
24
+ @base_indent = indent
25
+ @config = config
26
+ end
27
+
28
+ def build
29
+ s = @config['system'] && system_config_build(@config['system'])
30
+ c = @config['config'] && config_build(@config['config'], root: true)
31
+ RootBuilder.new(s, c)
32
+ end
33
+
34
+ private
35
+
36
+ def system_config_build(config)
37
+ section_build('system', config)
38
+ end
39
+
40
+ def config_build(config, indent: 0, root: false)
41
+ sb = SectionBodyBuilder.new(indent, root: root)
42
+ config.each do |c|
43
+ if (lc = c.delete('label'))
44
+ sb.add_section(label_build(lc, indent: indent))
45
+ end
46
+
47
+ if (sc = c.delete('source'))
48
+ sb.add_section(source_build(sc, indent: indent))
49
+ end
50
+
51
+ if (fc = c.delete('filter'))
52
+ sb.add_section(filter_build(fc, indent: indent))
53
+ end
54
+
55
+ if (mc = c.delete('match'))
56
+ sb.add_section(match_build(mc, indent: indent))
57
+ end
58
+
59
+ if (wc = c.delete('worker'))
60
+ sb.add_section(worker_build(wc, indent: indent))
61
+ end
62
+
63
+ included_sections_build(c, sb, indent: indent)
64
+ end
65
+
66
+ sb
67
+ end
68
+
69
+ def label_build(config, indent: 0)
70
+ config = config.dup
71
+ name = config.delete('$name')
72
+ c = config.delete('config')
73
+ SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name)
74
+ end
75
+
76
+ def worker_build(config, indent: 0)
77
+ config = config.dup
78
+ num = config.delete('$arg')
79
+ c = config.delete('config')
80
+ SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num)
81
+ end
82
+
83
+ def source_build(config, indent: 0)
84
+ section_build('source', config, indent: indent)
85
+ end
86
+
87
+ def filter_build(config, indent: 0)
88
+ config = config.dup
89
+ tag = config.delete('$tag')
90
+ if tag.is_a?(Array)
91
+ section_build('filter', config, indent: indent, arg: tag&.join(','))
92
+ else
93
+ section_build('filter', config, indent: indent, arg: tag)
94
+ end
95
+ end
96
+
97
+ def match_build(config, indent: 0)
98
+ config = config.dup
99
+ tag = config.delete('$tag')
100
+ if tag.is_a?(Array)
101
+ section_build('match', config, indent: indent, arg: tag&.join(','))
102
+ else
103
+ section_build('match', config, indent: indent, arg: tag)
104
+ end
105
+ end
106
+
107
+ def included_sections_build(config, section_builder, indent: 0)
108
+ config.each_entry do |e|
109
+ k = e.keys.first
110
+ cc = e.delete(k)
111
+ case k
112
+ when 'label'
113
+ section_builder.add_section(label_build(cc, indent: indent))
114
+ when 'worker'
115
+ section_builder.add_section(worker_build(cc, indent: indent))
116
+ when 'source'
117
+ section_builder.add_section(source_build(cc, indent: indent))
118
+ when 'filter'
119
+ section_builder.add_section(filter_build(cc, indent: indent))
120
+ when 'match'
121
+ section_builder.add_section(match_build(cc, indent: indent))
122
+ end
123
+ end
124
+ end
125
+
126
+ def section_build(name, config, indent: 0, arg: nil)
127
+ sb = SectionBodyBuilder.new(indent + @base_indent)
128
+
129
+ if (v = config.delete('$type'))
130
+ sb.add_line('@type', v)
131
+ end
132
+
133
+ if (v = config.delete('$label'))
134
+ sb.add_line('@label', v)
135
+ end
136
+
137
+ if (v = config.delete('$id'))
138
+ sb.add_line('@id', v)
139
+ end
140
+
141
+ config.each do |key, val|
142
+ if val.is_a?(Array)
143
+ val.each do |v|
144
+ sb.add_section(section_build(key, v, indent: indent + @base_indent))
145
+ end
146
+ elsif val.is_a?(Hash)
147
+ harg = val.delete('$arg')
148
+ if harg.is_a?(Array)
149
+ # To prevent to generate invalid configuration for arg.
150
+ # "arg" should be String object and concatenated by ","
151
+ # when two or more objects are specified there.
152
+ sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg&.join(',')))
153
+ else
154
+ sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg))
155
+ end
156
+ else
157
+ sb.add_line(key, val)
158
+ end
159
+ end
160
+
161
+ SectionBuilder.new(name, sb, indent, arg)
162
+ end
163
+ end
164
+ end
165
+ end
166
+ end
@@ -0,0 +1,107 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ module Config
19
+ module YamlParser
20
+ SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do
21
+ def to_s
22
+ indent = ' ' * indent_size
23
+
24
+ if arg && !arg.to_s.empty?
25
+ "#{indent}<#{name} #{arg}>\n#{body}\n#{indent}</#{name}>"
26
+ else
27
+ "#{indent}<#{name}>\n#{body}\n#{indent}</#{name}>"
28
+ end
29
+ end
30
+
31
+ def to_element
32
+ elem = body.to_element
33
+ elem.name = name
34
+ elem.arg = arg.to_s if arg
35
+ elem.v1_config = true
36
+ elem
37
+ end
38
+ end
39
+
40
+ class RootBuilder
41
+ def initialize(system, conf)
42
+ @system = system
43
+ @conf = conf
44
+ end
45
+
46
+ attr_reader :system, :conf
47
+
48
+ def to_element
49
+ Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten)
50
+ end
51
+
52
+ def to_s
53
+ s = StringIO.new(+'')
54
+ s.puts(@system.to_s) if @system
55
+ s.puts(@conf.to_s) if @conf
56
+
57
+ s.string
58
+ end
59
+ end
60
+
61
+ class SectionBodyBuilder
62
+ Row = Struct.new(:key, :value, :indent) do
63
+ def to_s
64
+ "#{indent}#{key} #{value}"
65
+ end
66
+ end
67
+
68
+ def initialize(indent, root: false)
69
+ @indent = ' ' * indent
70
+ @bodies = []
71
+ @root = root
72
+ end
73
+
74
+ def add_line(k, v)
75
+ @bodies << Row.new(k, v, @indent)
76
+ end
77
+
78
+ def add_section(section)
79
+ @bodies << section
80
+ end
81
+
82
+ def to_element
83
+ if @root
84
+ return @bodies.map(&:to_element)
85
+ end
86
+
87
+ not_section, section = @bodies.partition { |e| e.is_a?(Row) }
88
+ r = {}
89
+ not_section.each do |e|
90
+ v = e.value
91
+ r[e.key] = v.respond_to?(:to_element) ? v.to_element : v
92
+ end
93
+
94
+ if @root
95
+ section.map(&:to_element)
96
+ else
97
+ Fluent::Config::Element.new('', '', r, section.map(&:to_element))
98
+ end
99
+ end
100
+
101
+ def to_s
102
+ @bodies.map(&:to_s).join("\n")
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,56 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/yaml_parser/loader'
18
+ require 'fluent/config/yaml_parser/parser'
19
+ require 'pathname'
20
+
21
+ module Fluent
22
+ module Config
23
+ module YamlParser
24
+ def self.parse(path)
25
+ context = Kernel.binding
26
+
27
+ unless context.respond_to?(:use_nil)
28
+ context.define_singleton_method(:use_nil) do
29
+ raise Fluent::SetNil
30
+ end
31
+ end
32
+
33
+ unless context.respond_to?(:use_default)
34
+ context.define_singleton_method(:use_default) do
35
+ raise Fluent::SetDefault
36
+ end
37
+ end
38
+
39
+ unless context.respond_to?(:hostname)
40
+ context.define_singleton_method(:hostname) do
41
+ Socket.gethostname
42
+ end
43
+ end
44
+
45
+ unless context.respond_to?(:worker_id)
46
+ context.define_singleton_method(:worker_id) do
47
+ ENV['SERVERENGINE_WORKER_ID'] || ''
48
+ end
49
+ end
50
+
51
+ s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
52
+ Fluent::Config::YamlParser::Parser.new(s).build.to_element
53
+ end
54
+ end
55
+ end
56
+ end
data/lib/fluent/config.rb CHANGED
@@ -17,6 +17,7 @@
17
17
  require 'fluent/config/error'
18
18
  require 'fluent/config/element'
19
19
  require 'fluent/configurable'
20
+ require 'fluent/config/yaml_parser'
20
21
 
21
22
  module Fluent
22
23
  module Config
@@ -25,7 +26,18 @@ module Fluent
25
26
  # @param additional_config [String] config which is added to last of config body
26
27
  # @param use_v1_config [Bool] config is formatted with v1 or not
27
28
  # @return [Fluent::Config]
28
- def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
29
+ def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil)
30
+ if type == :guess
31
+ config_file_ext = File.extname(config_path)
32
+ if config_file_ext == '.yaml' || config_file_ext == '.yml'
33
+ type = :yaml
34
+ end
35
+ end
36
+
37
+ if type == :yaml || type == :yml
38
+ return Fluent::Config::YamlParser.parse(config_path)
39
+ end
40
+
29
41
  config_fname = File.basename(config_path)
30
42
  config_basedir = File.dirname(config_path)
31
43
  config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
@@ -36,6 +48,7 @@ module Fluent
36
48
  end
37
49
  s
38
50
  end
51
+
39
52
  Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
40
53
  end
41
54
 
@@ -116,6 +116,8 @@ module Fluent
116
116
  if callback = find_callback
117
117
  callback.call(es)
118
118
  end
119
+ rescue Pipeline::OutputError => e
120
+ @emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
119
121
  rescue => e
120
122
  @emit_error_handler.handle_emits_error(tag, es, e)
121
123
  end
@@ -161,6 +163,17 @@ module Fluent
161
163
  private
162
164
 
163
165
  class Pipeline
166
+
167
+ class OutputError < StandardError
168
+ attr_reader :internal_error
169
+ attr_reader :processed_es
170
+
171
+ def initialize(internal_error, processed_es)
172
+ @internal_error = internal_error
173
+ @processed_es = processed_es
174
+ end
175
+ end
176
+
164
177
  def initialize
165
178
  @filters = []
166
179
  @output = nil
@@ -178,7 +191,12 @@ module Fluent
178
191
 
179
192
  def emit_events(tag, es)
180
193
  processed = @optimizer.filter_stream(tag, es)
181
- @output.emit_events(tag, processed)
194
+
195
+ begin
196
+ @output.emit_events(tag, processed)
197
+ rescue => e
198
+ raise OutputError.new(e, processed)
199
+ end
182
200
  end
183
201
 
184
202
  class FilterOptimizer
@@ -70,7 +70,7 @@ module Fluent
70
70
  super
71
71
 
72
72
  @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
73
- @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
73
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_count", help_text: "Number of count emits")
74
74
  @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
75
75
  @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
76
76
  @enable_size_metrics = !!system_config.enable_size_metrics
@@ -72,8 +72,8 @@ module Fluent
72
72
 
73
73
  def string_safe_encoding(str)
74
74
  unless str.valid_encoding?
75
- log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
76
75
  str = str.scrub('?')
76
+ log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
77
77
  end
78
78
  yield str
79
79
  end
@@ -16,8 +16,8 @@
16
16
 
17
17
  module Fluent
18
18
  module FileWrapper
19
- def self.open(*args)
20
- io = WindowsFile.new(*args).io
19
+ def self.open(path, mode='r')
20
+ io = WindowsFile.new(path, mode).io
21
21
  if block_given?
22
22
  v = yield io
23
23
  io.close
@@ -35,116 +35,36 @@ module Fluent
35
35
  end
36
36
  end
37
37
 
38
- module WindowsFileExtension
39
- attr_reader :path
40
-
41
- def stat
42
- s = super
43
- s.instance_variable_set :@ino, @ino
44
- def s.ino; @ino; end
45
- s
46
- end
47
- end
48
-
49
- class Win32Error < StandardError
50
- require 'windows/error'
51
- include Windows::Error
52
-
53
- attr_reader :errcode, :msg
54
-
55
- WSABASEERR = 10000
56
-
57
- def initialize(errcode, msg = nil)
58
- @errcode = errcode
59
- @msg = msg
60
- end
61
-
62
- def format_english_message(errcode)
63
- buf = 0.chr * 260
64
- flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
65
- english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
66
- FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
67
- buf.force_encoding(Encoding.default_external).strip
68
- end
69
-
70
- def to_s
71
- msg = super
72
- msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
73
- msg << " - #{@msg}" if @msg
74
- msg
75
- end
76
-
77
- def inspect
78
- "#<#{to_s}>"
79
- end
80
-
81
- def ==(other)
82
- return false if other.class != Win32Error
83
- @errcode == other.errcode && @msg == other.msg
84
- end
85
-
86
- def wsaerr?
87
- @errcode >= WSABASEERR
88
- end
89
- end
90
-
91
- # To open and get stat with setting FILE_SHARE_DELETE
92
38
  class WindowsFile
93
39
  require 'windows/file'
94
- require 'windows/error'
95
40
  require 'windows/handle'
96
- require 'windows/nio'
97
41
 
98
- include Windows::Error
42
+ include File::Constants
99
43
  include Windows::File
100
44
  include Windows::Handle
101
- include Windows::NIO
102
-
103
- def initialize(path, mode='r', sharemode=FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE)
104
- @path = path
105
- @file_handle = INVALID_HANDLE_VALUE
106
- @mode = mode
107
45
 
46
+ attr_reader :io
108
47
 
109
- access, creationdisposition, seektoend = case mode.delete('b')
110
- when "r" ; [FILE_GENERIC_READ , OPEN_EXISTING, false]
111
- when "r+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS , false]
112
- when "w" ; [FILE_GENERIC_WRITE , CREATE_ALWAYS, false]
113
- when "w+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, CREATE_ALWAYS, false]
114
- when "a" ; [FILE_GENERIC_WRITE , OPEN_ALWAYS , true]
115
- when "a+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS , true]
116
- else raise "unknown mode '#{mode}'"
117
- end
118
-
119
- @file_handle = CreateFile.call(@path, access, sharemode,
120
- 0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
121
- if @file_handle == INVALID_HANDLE_VALUE
122
- win32err = Win32Error.new(Win32::API.last_error, path)
123
- errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
124
- if errno == Errno::EINVAL::Errno || win32err.wsaerr?
125
- # maybe failed to map
126
- raise win32err
127
- else
128
- raise SystemCallError.new(win32err.message, errno)
129
- end
48
+ def initialize(path, mode='r')
49
+ @path = path
50
+ @io = File.open(path, mode2flags(mode))
51
+ @file_handle = _get_osfhandle(@io.to_i)
52
+ @io.instance_variable_set(:@file_index, self.ino)
53
+ def @io.ino
54
+ @file_index
130
55
  end
131
56
  end
132
57
 
133
58
  def close
134
- CloseHandle.call(@file_handle)
59
+ @io.close
135
60
  @file_handle = INVALID_HANDLE_VALUE
136
61
  end
137
62
 
138
- def io
139
- fd = _open_osfhandle(@file_handle, 0)
140
- raise Errno::ENOENT if fd == -1
141
- io = File.for_fd(fd, @mode)
142
- io.instance_variable_set :@ino, self.ino
143
- io.instance_variable_set :@path, @path
144
- io.extend WindowsFileExtension
145
- io
146
- end
147
-
63
+ # To keep backward compatibility, we continue to use GetFileInformationByHandle()
64
+ # to get file id.
65
+ # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
66
+ # and returned value is different with above one, former one is 64 bit while
67
+ # later one is 128bit.
148
68
  def ino
149
69
  by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4) #72bytes
150
70
 
@@ -155,6 +75,41 @@ module Fluent
155
75
  by_handle_file_information.unpack("I11Q1")[11] # fileindex
156
76
  end
157
77
 
78
+ def stat
79
+ raise Errno::ENOENT if delete_pending
80
+ s = File.stat(@path)
81
+ s.instance_variable_set :@ino, self.ino
82
+ def s.ino; @ino; end
83
+ s
84
+ end
85
+
86
+ private
87
+
88
+ def mode2flags(mode)
89
+ # Always inject File::Constants::SHARE_DELETE
90
+ # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
91
+ # To enable SHARE_DELETE, BINARY is also required.
92
+ # https://bugs.ruby-lang.org/issues/11218
93
+ # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
94
+ flags = BINARY | SHARE_DELETE
95
+ case mode.delete("b")
96
+ when "r"
97
+ flags |= RDONLY
98
+ when "r+"
99
+ flags |= RDWR
100
+ when "w"
101
+ flags |= WRONLY | CREAT | TRUNC
102
+ when "w+"
103
+ flags |= RDWR | CREAT | TRUNC
104
+ when "a"
105
+ flags |= WRONLY | CREAT | APPEND
106
+ when "a+"
107
+ flags |= RDWR | CREAT | APPEND
108
+ else
109
+ raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
110
+ end
111
+ end
112
+
158
113
  # DeletePending is a Windows-specific file state that roughly means
159
114
  # "this file is queued for deletion, so close any open handlers"
160
115
  #
@@ -173,15 +128,5 @@ module Fluent
173
128
 
174
129
  return buf.unpack("QQICC")[3] != 0
175
130
  end
176
-
177
- private :delete_pending
178
-
179
- def stat
180
- raise Errno::ENOENT if delete_pending
181
- s = File.stat(@path)
182
- s.instance_variable_set :@ino, self.ino
183
- def s.ino; @ino; end
184
- s
185
- end
186
131
  end
187
132
  end if Fluent.windows?
@@ -40,7 +40,7 @@ module Fluent::Plugin
40
40
  config_param :backlog, :integer, default: nil
41
41
  # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
42
42
  desc 'The timeout time used to set linger option.'
43
- config_param :linger_timeout, :integer, default: 0
43
+ config_param :linger_timeout, :integer, default: nil, deprecated: "use transport directive"
44
44
  # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
45
45
  config_param :blocking_timeout, :time, default: 0.5
46
46
  desc 'Try to resolve hostname from IP addresses or not.'
@@ -314,8 +314,16 @@ module Fluent::Plugin
314
314
  @parser_json.parse(js) do |_time, record|
315
315
  return nil, record
316
316
  end
317
+ elsif ndjson = params['ndjson']
318
+ events = []
319
+ ndjson.split(/\r?\n/).each do |js|
320
+ @parser_json.parse(js) do |_time, record|
321
+ events.push(record)
322
+ end
323
+ end
324
+ return nil, events
317
325
  else
318
- raise "'json' or 'msgpack' parameter is required"
326
+ raise "'json', 'ndjson' or 'msgpack' parameter is required"
319
327
  end
320
328
  end
321
329
 
@@ -567,6 +575,8 @@ module Fluent::Plugin
567
575
  params['json'] = @body
568
576
  elsif @content_type =~ /^application\/msgpack/
569
577
  params['msgpack'] = @body
578
+ elsif @content_type =~ /^application\/x-ndjson/
579
+ params['ndjson'] = @body
570
580
  end
571
581
  path_info = uri.path
572
582