fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -22,54 +22,168 @@ module Fluent::Plugin
22
22
  class GrepFilter < Filter
23
23
  Fluent::Plugin.register_filter('grep', self)
24
24
 
25
- REGEXP_MAX_NUM = 20
25
+ def initialize
26
+ super
26
27
 
27
- (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}", :string, default: nil }
28
- (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
28
+ @_regexp_and_conditions = nil
29
+ @_exclude_and_conditions = nil
30
+ @_regexp_or_conditions = nil
31
+ @_exclude_or_conditions = nil
32
+ end
29
33
 
30
34
  # for test
31
- attr_reader :regexps
32
- attr_reader :excludes
35
+ attr_reader :_regexp_and_conditions, :_exclude_and_conditions, :_regexp_or_conditions, :_exclude_or_conditions
36
+
37
+ helpers :record_accessor
38
+
39
+ REGEXP_MAX_NUM = 20
40
+
41
+ (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}", :string, default: nil, deprecated: "Use <regexp> section" }
42
+ (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil, deprecated: "Use <exclude> section" }
43
+
44
+ config_section :regexp, param_name: :regexps, multi: true do
45
+ desc "The field name to which the regular expression is applied."
46
+ config_param :key, :string
47
+ desc "The regular expression."
48
+ config_param :pattern, :regexp
49
+ end
50
+
51
+ config_section :exclude, param_name: :excludes, multi: true do
52
+ desc "The field name to which the regular expression is applied."
53
+ config_param :key, :string
54
+ desc "The regular expression."
55
+ config_param :pattern, :regexp
56
+ end
57
+
58
+ config_section :and, param_name: :and_conditions, multi: true do
59
+ config_section :regexp, param_name: :regexps, multi: true do
60
+ desc "The field name to which the regular expression is applied."
61
+ config_param :key, :string
62
+ desc "The regular expression."
63
+ config_param :pattern, :regexp
64
+ end
65
+ config_section :exclude, param_name: :excludes, multi: true do
66
+ desc "The field name to which the regular expression is applied."
67
+ config_param :key, :string
68
+ desc "The regular expression."
69
+ config_param :pattern, :regexp
70
+ end
71
+ end
72
+
73
+ config_section :or, param_name: :or_conditions, multi: true do
74
+ config_section :regexp, param_name: :regexps, multi: true do
75
+ desc "The field name to which the regular expression is applied."
76
+ config_param :key, :string
77
+ desc "The regular expression."
78
+ config_param :pattern, :regexp
79
+ end
80
+ config_section :exclude, param_name: :excludes, multi: true do
81
+ desc "The field name to which the regular expression is applied."
82
+ config_param :key, :string
83
+ desc "The regular expression."
84
+ config_param :pattern, :regexp
85
+ end
86
+ end
33
87
 
34
88
  def configure(conf)
35
89
  super
36
90
 
37
- @regexps = {}
91
+ regexp_and_conditions = {}
92
+ regexp_or_conditions = {}
93
+ exclude_and_conditions = {}
94
+ exclude_or_conditions = {}
95
+
38
96
  (1..REGEXP_MAX_NUM).each do |i|
39
97
  next unless conf["regexp#{i}"]
40
98
  key, regexp = conf["regexp#{i}"].split(/ /, 2)
41
99
  raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
42
- raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
43
- @regexps[key] = Regexp.compile(regexp)
100
+ raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if regexp_and_conditions[key]
101
+ regexp_and_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(regexp))
44
102
  end
45
103
 
46
- @excludes = {}
47
104
  (1..REGEXP_MAX_NUM).each do |i|
48
105
  next unless conf["exclude#{i}"]
49
106
  key, exclude = conf["exclude#{i}"].split(/ /, 2)
50
107
  raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
51
- raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
52
- @excludes[key] = Regexp.compile(exclude)
108
+ raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if exclude_or_conditions[key]
109
+ exclude_or_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(exclude))
110
+ end
111
+
112
+ if @regexps.size > 1
113
+ log.info "Top level multiple <regexp> is intepreted as 'and' condition"
53
114
  end
115
+ @regexps.each do |e|
116
+ raise Fluent::ConfigError, "Duplicate key: #{e.key}" if regexp_and_conditions.key?(e.key)
117
+ regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
118
+ end
119
+
120
+ if @excludes.size > 1
121
+ log.info "Top level multiple <exclude> is intepreted as 'or' condition"
122
+ end
123
+ @excludes.each do |e|
124
+ raise Fluent::ConfigError, "Duplicate key: #{e.key}" if exclude_or_conditions.key?(e.key)
125
+ exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
126
+ end
127
+
128
+ @and_conditions.each do |and_condition|
129
+ if !and_condition.regexps.empty? && !and_condition.excludes.empty?
130
+ raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <and>"
131
+ end
132
+ and_condition.regexps.each do |e|
133
+ raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if regexp_and_conditions.key?(e.key)
134
+ regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
135
+ end
136
+ and_condition.excludes.each do |e|
137
+ raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if exclude_and_conditions.key?(e.key)
138
+ exclude_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
139
+ end
140
+ end
141
+
142
+ @or_conditions.each do |or_condition|
143
+ if !or_condition.regexps.empty? && !or_condition.excludes.empty?
144
+ raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <or>"
145
+ end
146
+ or_condition.regexps.each do |e|
147
+ raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if regexp_or_conditions.key?(e.key)
148
+ regexp_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
149
+ end
150
+ or_condition.excludes.each do |e|
151
+ raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if exclude_or_conditions.key?(e.key)
152
+ exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
153
+ end
154
+ end
155
+
156
+ @_regexp_and_conditions = regexp_and_conditions.values unless regexp_and_conditions.empty?
157
+ @_exclude_and_conditions = exclude_and_conditions.values unless exclude_and_conditions.empty?
158
+ @_regexp_or_conditions = regexp_or_conditions.values unless regexp_or_conditions.empty?
159
+ @_exclude_or_conditions = exclude_or_conditions.values unless exclude_or_conditions.empty?
54
160
  end
55
161
 
56
162
  def filter(tag, time, record)
57
- result = nil
58
163
  begin
59
- catch(:break_loop) do
60
- @regexps.each do |key, regexp|
61
- throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
62
- end
63
- @excludes.each do |key, exclude|
64
- throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
65
- end
66
- result = record
164
+ if @_regexp_and_conditions && @_regexp_and_conditions.any? { |expression| !expression.match?(record) }
165
+ return nil
166
+ end
167
+ if @_regexp_or_conditions && @_regexp_or_conditions.none? { |expression| expression.match?(record) }
168
+ return nil
169
+ end
170
+ if @_exclude_and_conditions && @_exclude_and_conditions.all? { |expression| expression.match?(record) }
171
+ return nil
172
+ end
173
+ if @_exclude_or_conditions && @_exclude_or_conditions.any? { |expression| expression.match?(record) }
174
+ return nil
67
175
  end
68
176
  rescue => e
69
177
  log.warn "failed to grep events", error: e
70
178
  log.warn_backtrace
71
179
  end
72
- result
180
+ record
181
+ end
182
+
183
+ Expression = Struct.new(:key, :pattern) do
184
+ def match?(record)
185
+ ::Fluent::StringUtil.match_regexp(pattern, key.call(record).to_s)
186
+ end
73
187
  end
74
188
  end
75
189
  end
@@ -24,14 +24,23 @@ module Fluent::Plugin
24
24
  class ParserFilter < Filter
25
25
  Fluent::Plugin.register_filter('parser', self)
26
26
 
27
- helpers :parser, :compat_parameters
27
+ helpers :parser, :record_accessor, :compat_parameters
28
28
 
29
+ desc 'Specify field name in the record to parse.'
29
30
  config_param :key_name, :string
31
+ desc 'Keep original key-value pair in parsed result.'
30
32
  config_param :reserve_data, :bool, default: false
33
+ desc 'Keep original event time in parsed result.'
31
34
  config_param :reserve_time, :bool, default: false
35
+ desc 'Remove "key_name" field from the record when parsing is succeeded'
36
+ config_param :remove_key_name_field, :bool, default: false
37
+ desc 'Store parsed values with specified key name prefix.'
32
38
  config_param :inject_key_prefix, :string, default: nil
39
+ desc 'If true, invalid string is replaced with safe characters and re-parse it.'
33
40
  config_param :replace_invalid_sequence, :bool, default: false
41
+ desc 'Store parsed values as a hash value in a field.'
34
42
  config_param :hash_value_field, :string, default: nil
43
+ desc 'Emit invalid record to @ERROR label'
35
44
  config_param :emit_invalid_record_to_error, :bool, default: true
36
45
 
37
46
  attr_reader :parser
@@ -41,6 +50,7 @@ module Fluent::Plugin
41
50
 
42
51
  super
43
52
 
53
+ @accessor = record_accessor_create(@key_name)
44
54
  @parser = parser_create
45
55
  end
46
56
 
@@ -48,7 +58,7 @@ module Fluent::Plugin
48
58
  REPLACE_CHAR = '?'.freeze
49
59
 
50
60
  def filter_with_time(tag, time, record)
51
- raw_value = record[@key_name]
61
+ raw_value = @accessor.call(record)
52
62
  if raw_value.nil?
53
63
  if @emit_invalid_record_to_error
54
64
  router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
@@ -67,6 +77,7 @@ module Fluent::Plugin
67
77
  else
68
78
  t.nil? ? time : t
69
79
  end
80
+ @accessor.delete(record) if @remove_key_name_field
70
81
  r = handle_parsed(tag, record, t, values)
71
82
  return t, r
72
83
  else
@@ -27,10 +27,12 @@ module Fluent::Plugin
27
27
  class RecordTransformerFilter < Fluent::Plugin::Filter
28
28
  Fluent::Plugin.register_filter('record_transformer', self)
29
29
 
30
+ helpers :record_accessor
31
+
30
32
  desc 'A comma-delimited list of keys to delete.'
31
- config_param :remove_keys, :string, default: nil
33
+ config_param :remove_keys, :array, default: nil
32
34
  desc 'A comma-delimited list of keys to keep.'
33
- config_param :keep_keys, :string, default: nil
35
+ config_param :keep_keys, :array, default: nil
34
36
  desc 'Create new Hash to transform incoming data'
35
37
  config_param :renew_record, :bool, default: false
36
38
  desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
@@ -52,15 +54,14 @@ module Fluent::Plugin
52
54
  end
53
55
  end
54
56
 
55
- if @remove_keys
56
- @remove_keys = @remove_keys.split(',')
57
- end
58
-
59
57
  if @keep_keys
60
58
  raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
61
- @keep_keys = @keep_keys.split(',')
62
59
  end
63
60
 
61
+ @key_deleters = if @remove_keys
62
+ @remove_keys.map { |k| record_accessor_create(k) }
63
+ end
64
+
64
65
  placeholder_expander_params = {
65
66
  log: log,
66
67
  auto_typecast: @auto_typecast,
@@ -101,8 +102,7 @@ module Fluent::Plugin
101
102
  if @renew_time_key && new_record.has_key?(@renew_time_key)
102
103
  time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
103
104
  end
104
- @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
105
-
105
+ @key_deleters.each { |deleter| deleter.delete(new_record) } if @key_deleters
106
106
  new_es.add(time, new_record)
107
107
  rescue => e
108
108
  router.emit_error_event(tag, time, record, e)
@@ -129,7 +129,9 @@ module Fluent::Plugin
129
129
  placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
130
130
 
131
131
  new_record = @renew_record ? {} : record.dup
132
- @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
132
+ @keep_keys.each do |k|
133
+ new_record[k] = record[k] if record.has_key?(k)
134
+ end if @keep_keys && @renew_record
133
135
  new_record.merge!(expand_placeholders(@map, placeholders))
134
136
 
135
137
  new_record
@@ -218,7 +220,7 @@ module Fluent::Plugin
218
220
  # @param [String] str
219
221
  # @param [Boolean] force_stringify the value must be string, used for hash key
220
222
  def expand(str, placeholders, force_stringify = false)
221
- if @auto_typecast and !force_stringify
223
+ if @auto_typecast && !force_stringify
222
224
  single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
223
225
  if single_placeholder_matched
224
226
  log_if_unknown_placeholder($1, placeholders)
@@ -261,9 +263,9 @@ module Fluent::Plugin
261
263
  def preprocess_map(value, force_stringify = false)
262
264
  new_value = nil
263
265
  if value.is_a?(String)
264
- if @auto_typecast and !force_stringify
266
+ if @auto_typecast && !force_stringify
265
267
  num_placeholders = value.scan('${').size
266
- if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
268
+ if num_placeholders == 1 && value.start_with?('${') && value.end_with?('}')
267
269
  new_value = value[2..-2] # ${..} => ..
268
270
  end
269
271
  end
@@ -314,7 +316,7 @@ module Fluent::Plugin
314
316
  end
315
317
 
316
318
  (Object.instance_methods).each do |m|
317
- undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
319
+ undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
318
320
  end
319
321
  end
320
322
  end
@@ -21,13 +21,14 @@ module Fluent
21
21
  class StdoutFormatter < Formatter
22
22
  Plugin.register_formatter('stdout', self)
23
23
 
24
- TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
24
+ TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%N %z'
25
25
 
26
26
  config_param :output_type, :string, default: 'json'
27
27
 
28
28
  def configure(conf)
29
29
  super
30
30
 
31
+ @time_formatter = Strftime.new(TIME_FORMAT)
31
32
  @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
32
33
  @sub_formatter.configure(conf)
33
34
  end
@@ -38,7 +39,7 @@ module Fluent
38
39
  end
39
40
 
40
41
  def format(tag, time, record)
41
- "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
42
+ "#{@time_formatter.exec(Time.at(time).localtime)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
42
43
  end
43
44
 
44
45
  def stop
@@ -25,9 +25,13 @@ module Fluent
25
25
  config_param :keys, :array, value_type: :string
26
26
  desc 'The delimiter character (or string) of TSV values'
27
27
  config_param :delimiter, :string, default: "\t"
28
+ desc 'The parameter to enable writing to new lines'
29
+ config_param :add_newline, :bool, default: true
28
30
 
29
31
  def format(tag, time, record)
30
- @keys.map{|k| record[k].to_s }.join(@delimiter)
32
+ formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
33
+ formatted << "\n".freeze if @add_newline
34
+ formatted
31
35
  end
32
36
  end
33
37
  end
@@ -34,6 +34,9 @@ module Fluent::Plugin
34
34
 
35
35
  def configure(conf)
36
36
  super
37
+ if system_config.workers > 1
38
+ @port += fluentd_worker_id
39
+ end
37
40
  if @unix_path
38
41
  unless ::Fluent::FileUtil.writable?(@unix_path)
39
42
  raise Fluent::ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
@@ -41,6 +44,10 @@ module Fluent::Plugin
41
44
  end
42
45
  end
43
46
 
47
+ def multi_workers_ready?
48
+ @unix_path.nil?
49
+ end
50
+
44
51
  def start
45
52
  super
46
53
 
@@ -50,7 +57,7 @@ module Fluent::Plugin
50
57
  else
51
58
  uri = "druby://#{@bind}:#{@port}"
52
59
  end
53
- log.info "listening dRuby", uri: uri, object: @object
60
+ log.info "listening dRuby", uri: uri, object: @object, worker: fluentd_worker_id
54
61
  obj = eval(@object)
55
62
  @server = DRb::DRbServer.new(uri, obj)
56
63
  end
@@ -199,8 +199,8 @@ module Fluent::Plugin
199
199
  when :pingpong
200
200
  success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
201
201
  unless success
202
+ conn.on(:write_complete) { |c| c.close_after_write_complete }
202
203
  send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
203
- conn.close
204
204
  next
205
205
  end
206
206
  send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))
@@ -78,10 +78,16 @@ module Fluent::Plugin
78
78
  @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
79
79
  @parser_json.estimate_current_event = false
80
80
  @format_name = 'default'
81
+ @parser_time_key = if parser_config = conf.elements('parse').first
82
+ parser_config['time_key'] || 'time'
83
+ else
84
+ 'time'
85
+ end
81
86
  method(:parse_params_default)
82
87
  else
83
88
  @parser = parser_create
84
89
  @format_name = @parser_configs.first['@type']
90
+ @parser_time_key = @parser.time_key
85
91
  method(:parse_params_with_parser)
86
92
  end
87
93
  self.singleton_class.module_eval do
@@ -202,7 +208,18 @@ module Fluent::Plugin
202
208
  if @add_remote_addr
203
209
  single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
204
210
  end
205
- single_time = single_record.delete("time") || time
211
+
212
+ if defined? @parser
213
+ single_time = @parser.parse_time(single_record)
214
+ single_time, single_record = @parser.convert_values(single_time, single_record)
215
+ else
216
+ single_time = if t = single_record.delete(@parser_time_key)
217
+ Fluent::EventTime.from_time(Time.at(t))
218
+ else
219
+ time
220
+ end
221
+ end
222
+
206
223
  mes.add(single_time, single_record)
207
224
  end
208
225
  router.emit_stream(tag, mes)
@@ -301,6 +318,7 @@ module Fluent::Plugin
301
318
  end
302
319
  @env = {}
303
320
  @content_type = ""
321
+ @content_encoding = ""
304
322
  headers.each_pair {|k,v|
305
323
  @env["HTTP_#{k.gsub('-','_').upcase}"] = v
306
324
  case k
@@ -310,6 +328,8 @@ module Fluent::Plugin
310
328
  size = v.to_i
311
329
  when /Content-Type/i
312
330
  @content_type = v
331
+ when /Content-Encoding/i
332
+ @content_encoding = v
313
333
  when /Connection/i
314
334
  if v =~ /close/i
315
335
  @keep_alive = false
@@ -322,6 +342,10 @@ module Fluent::Plugin
322
342
  # For multiple X-Forwarded-For headers. Use first header value.
323
343
  v = v.first if v.is_a?(Array)
324
344
  @remote_addr = v.split(",").first
345
+ when /Access-Control-Request-Method/i
346
+ @access_control_request_method = v
347
+ when /Access-Control-Request-Headers/i
348
+ @access_control_request_headers = v
325
349
  end
326
350
  }
327
351
  if expect
@@ -347,20 +371,70 @@ module Fluent::Plugin
347
371
  @body << chunk
348
372
  end
349
373
 
374
+ # Web browsers can send an OPTIONS request before performing POST
375
+ # to check if cross-origin requests are supported.
376
+ def handle_options_request
377
+ # Is CORS enabled in the first place?
378
+ if @cors_allow_origins.nil?
379
+ return send_response_and_close("403 Forbidden", {}, "")
380
+ end
381
+
382
+ # in_http does not support HTTP methods except POST
383
+ if @access_control_request_method != 'POST'
384
+ return send_response_and_close("403 Forbidden", {}, "")
385
+ end
386
+
387
+ header = {
388
+ "Access-Control-Allow-Methods" => "POST",
389
+ "Access-Control-Allow-Headers" => @access_control_request_headers || "",
390
+ }
391
+
392
+ # Check the origin and send back a CORS response
393
+ if @cors_allow_origins.include?('*')
394
+ header["Access-Control-Allow-Origin"] = "*"
395
+ send_response_and_close("200 OK", header, "")
396
+ elsif @cors_allow_origins.include?(@origin)
397
+ header["Access-Control-Allow-Origin"] = @origin
398
+ send_response_and_close("200 OK", header, "")
399
+ else
400
+ send_response_and_close("403 Forbidden", {}, "")
401
+ end
402
+ end
403
+
350
404
  def on_message_complete
351
405
  return if closing?
352
406
 
407
+ if @parser.http_method == 'OPTIONS'
408
+ return handle_options_request()
409
+ end
410
+
353
411
  # CORS check
354
412
  # ==========
355
413
  # For every incoming request, we check if we have some CORS
356
414
  # restrictions and white listed origins through @cors_allow_origins.
357
415
  unless @cors_allow_origins.nil?
358
- unless @cors_allow_origins.include?(@origin)
416
+ unless @cors_allow_origins.include?('*') or @cors_allow_origins.include?(@origin)
359
417
  send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
360
418
  return
361
419
  end
362
420
  end
363
421
 
422
+ # Content Encoding
423
+ # =================
424
+ # Decode payload according to the "Content-Encoding" header.
425
+ # For now, we only support 'gzip' and 'deflate'.
426
+ begin
427
+ if @content_encoding == 'gzip'
428
+ @body = Zlib::GzipReader.new(StringIO.new(@body)).read
429
+ elsif @content_encoding == 'deflate'
430
+ @body = Zlib::Inflate.inflate(@body)
431
+ end
432
+ rescue
433
+ @log.warn 'fails to decode payload', error: $!.to_s
434
+ send_response_and_close("400 Bad Request", {}, "")
435
+ return
436
+ end
437
+
364
438
  @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
365
439
 
366
440
  uri = URI.parse(@parser.request_url)
@@ -386,7 +460,14 @@ module Fluent::Plugin
386
460
  code, header, body = *@callback.call(path_info, params)
387
461
  body = body.to_s
388
462
 
389
- header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
463
+ unless @cors_allow_origins.nil?
464
+ if @cors_allow_origins.include?('*')
465
+ header['Access-Control-Allow-Origin'] = '*'
466
+ elsif @cors_allow_origins.include?(@origin)
467
+ header['Access-Control-Allow-Origin'] = @origin
468
+ end
469
+ end
470
+
390
471
  if @keep_alive
391
472
  header['Connection'] = 'Keep-Alive'
392
473
  send_response(code, header, body)