fluentd 1.11.1-x64-mingw32 → 1.11.2-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -0
  3. data/example/copy_roundrobin.conf +3 -3
  4. data/example/counter.conf +1 -1
  5. data/example/filter_stdout.conf +2 -2
  6. data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
  7. data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
  8. data/example/logevents.conf +5 -5
  9. data/example/multi_filters.conf +1 -1
  10. data/example/out_exec_filter.conf +2 -2
  11. data/example/out_forward.conf +1 -1
  12. data/example/out_forward_buf_file.conf +1 -1
  13. data/example/out_forward_client.conf +5 -5
  14. data/example/out_forward_heartbeat_none.conf +1 -1
  15. data/example/out_forward_sd.conf +1 -1
  16. data/example/out_forward_shared_key.conf +2 -2
  17. data/example/out_forward_tls.conf +1 -1
  18. data/example/out_forward_users.conf +3 -3
  19. data/example/out_null.conf +4 -4
  20. data/example/secondary_file.conf +1 -1
  21. data/lib/fluent/match.rb +9 -0
  22. data/lib/fluent/plugin/buffer.rb +24 -4
  23. data/lib/fluent/plugin/in_dummy.rb +2 -123
  24. data/lib/fluent/plugin/in_http.rb +2 -2
  25. data/lib/fluent/plugin/in_sample.rb +141 -0
  26. data/lib/fluent/plugin_helper/cert_option.rb +5 -8
  27. data/lib/fluent/plugin_helper/socket.rb +1 -1
  28. data/lib/fluent/supervisor.rb +1 -1
  29. data/lib/fluent/version.rb +1 -1
  30. data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
  31. data/test/plugin_helper/data/cert/empty.pem +0 -0
  32. data/test/plugin_helper/test_cert_option.rb +7 -0
  33. data/test/plugin_helper/test_server.rb +34 -0
  34. data/test/plugin_helper/test_socket.rb +8 -0
  35. data/test/test_match.rb +11 -0
  36. data/test/test_static_config_analysis.rb +2 -2
  37. metadata +9 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: de31e05af11ea6a1c1c8a438c174992bbd2f68119a4a0f912238448b591f9dc4
4
- data.tar.gz: f446b2552386447d237c83c0512ec11914761fda1b959c7ef695deff150d2969
3
+ metadata.gz: 783193e143e478f7d0688b10b7f295c4d84a25814caceb52f0b683da3519b965
4
+ data.tar.gz: 809c4084ce6e1323e708561d0b64d30dd3ac7baf20bfe6becece0759b59c43e7
5
5
  SHA512:
6
- metadata.gz: 5ed9f43e5efaabde10dfa2ef3f85d7ca3ffc05f0fd2019c5a63105587728f779739d2a9f7724d6a0fbb4997fd9dbcaf08bb48d1e2b73e1a03b99fafa2a389b1c
7
- data.tar.gz: '09ff256fcc45b357443ee47e1a590b569ae4a11cd3c0c2b05a48f3288498822b1124eb3e4b0168023e8ffb6796a4292e641d6011daef62f115965cff046d2290'
6
+ metadata.gz: aad9889623e4976c9c84892d8290d15ab626c7ab1d1ea5977fe9752704c7bc6c462e8bd0e8e0dd79562a5bad9bc43f0a8fd7b1f633615d1da64de9285e08e38a
7
+ data.tar.gz: 1f3930325ae5fc28a632e271d1d240bb2d14a333dd0d441d3217e22e278ab968a005ccd463e106f17fb82a8ecb1f8e58842774d48c25c978840f013a46193a3a
@@ -1,5 +1,27 @@
1
1
  # v1.11
2
2
 
3
+ ## Release v1.11.2 - 2020/08/04
4
+
5
+ ### Enhancement
6
+
7
+ * `in_dummy` renamed to `in_sample`
8
+ https://github.com/fluent/fluentd/pull/3065
9
+ * Allow regular expression in filter/match directive
10
+ https://github.com/fluent/fluentd/pull/3071
11
+ * Refactoring code
12
+ https://github.com/fluent/fluentd/pull/3051
13
+
14
+ ### Bug fix
15
+
16
+ * buffer: Fix log message for `chunk_limit_records` case
17
+ https://github.com/fluent/fluentd/pull/3079
18
+ * buffer: Fix timekey optimization for non-windows platform
19
+ https://github.com/fluent/fluentd/pull/3092
20
+ * cert: Raise an error for broken certificate file
21
+ https://github.com/fluent/fluentd/pull/3086
22
+ * cert: Set TLS ciphers list correcty on older OpenSSL
23
+ https://github.com/fluent/fluentd/pull/3093
24
+
3
25
  ## Release v1.11.1 - 2020/06/22
4
26
 
5
27
  ### Enhancement
@@ -1,12 +1,12 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  @label @test
4
4
  tag test.copy
5
5
  auto_increment_key id
6
6
  </source>
7
7
 
8
8
  <source>
9
- @type dummy
9
+ @type sample
10
10
  @label @test
11
11
  tag test.rr
12
12
  auto_increment_key id
@@ -36,4 +36,4 @@
36
36
  output_type ltsv
37
37
  </store>
38
38
  </match>
39
- </label>
39
+ </label>
@@ -8,7 +8,7 @@
8
8
  </system>
9
9
 
10
10
  <source>
11
- @type dummy
11
+ @type sample
12
12
  tag "test.data"
13
13
  auto_increment_key number
14
14
  </source>
@@ -1,6 +1,6 @@
1
1
  <source>
2
- @type dummy
3
- tag dummy
2
+ @type sample
3
+ tag sample
4
4
  </source>
5
5
 
6
6
  <filter **>
@@ -1,11 +1,11 @@
1
1
  <source>
2
- @type dummy
3
- tag dummy
2
+ @type sample
3
+ tag sample
4
4
  rate 100
5
- dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
5
+ sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
6
6
  </source>
7
7
 
8
- <match dummy>
8
+ <match sample>
9
9
  @type null
10
10
  never_flush true
11
11
  <buffer>
@@ -1,16 +1,16 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  @label @main
4
4
  tag "test.data"
5
5
  size 2
6
6
  rate 10
7
- dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
7
+ sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
8
8
  auto_increment_key number
9
9
  </source>
10
10
 
11
11
  <label @main>
12
12
  <match test.data>
13
- @type buffered_stdout
13
+ @type stdout
14
14
  <buffer>
15
15
  @type file
16
16
  path "#{Dir.pwd}/compressed_buffers"
@@ -1,10 +1,10 @@
1
1
  <source>
2
- @type dummy
3
- @label @dummylog
2
+ @type sample
3
+ @label @samplelog
4
4
  tag "data"
5
- dummy {"message":"yay"}
5
+ sample {"message":"yay"}
6
6
  </source>
7
- <label @dummylog>
7
+ <label @samplelog>
8
8
  <match **>
9
9
  @type stdout
10
10
  </match>
@@ -22,4 +22,4 @@
22
22
  # hostname_key "host"
23
23
  # </inject>
24
24
  # </match>
25
- </label>
25
+ </label>
@@ -1,7 +1,7 @@
1
1
  # This example is to measure optimized filter pipeline performance.
2
2
 
3
3
  <source>
4
- @type dummy
4
+ @type sample
5
5
  tag test
6
6
  size 1000
7
7
  </source>
@@ -1,10 +1,10 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  @label @exec
4
4
  tag exec_input
5
5
  rate 10
6
6
  auto_increment_key num
7
- dummy {"data":"mydata"}
7
+ sample {"data":"mydata"}
8
8
  </source>
9
9
 
10
10
  <label @exec>
@@ -1,5 +1,5 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
 
@@ -1,5 +1,5 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
 
@@ -1,21 +1,21 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
  <source>
6
- @type dummy
6
+ @type sample
7
7
  tag test2
8
8
  </source>
9
9
  <source>
10
- @type dummy
10
+ @type sample
11
11
  tag test3
12
12
  </source>
13
13
  <source>
14
- @type dummy
14
+ @type sample
15
15
  tag test4
16
16
  </source>
17
17
  <source>
18
- @type dummy
18
+ @type sample
19
19
  tag test5
20
20
  </source>
21
21
 
@@ -1,5 +1,5 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
 
@@ -1,5 +1,5 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
 
@@ -1,9 +1,9 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
  <source>
6
- @type dummy
6
+ @type sample
7
7
  tag test2
8
8
  </source>
9
9
 
@@ -1,5 +1,5 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
 
@@ -1,13 +1,13 @@
1
1
  <source>
2
- @type dummy
2
+ @type sample
3
3
  tag test
4
4
  </source>
5
5
  <source>
6
- @type dummy
6
+ @type sample
7
7
  tag test2
8
8
  </source>
9
9
  <source>
10
- @type dummy
10
+ @type sample
11
11
  tag test3
12
12
  </source>
13
13
 
@@ -2,17 +2,17 @@
2
2
  # bundle exec bin/fluentd -c example/out_buffered_null.conf
3
3
  # (+ --emit-error-log-interval 10)
4
4
  <source>
5
- @type dummy
6
- tag dummy
5
+ @type sample
6
+ tag sample
7
7
  rate 500000000
8
- dummy [
8
+ sample [
9
9
  {"message": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
10
10
  {"message": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},
11
11
  {"message": "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}
12
12
  ]
13
13
  </source>
14
14
 
15
- <match dummy.**>
15
+ <match sample.**>
16
16
  @type null
17
17
  <buffer>
18
18
  flush_interval 60s
@@ -3,7 +3,7 @@
3
3
  </system>
4
4
 
5
5
  <source>
6
- @type dummy
6
+ @type sample
7
7
  tag test
8
8
  </source>
9
9
 
@@ -33,6 +33,15 @@ module Fluent
33
33
 
34
34
  class GlobMatchPattern < MatchPattern
35
35
  def initialize(pat)
36
+ if pat.start_with?('/')
37
+ if pat.end_with?('/')
38
+ @regex = Regexp.new("\\A"+pat[1..-2]+"\\Z")
39
+ return
40
+ else
41
+ raise Fluent::ConfigError, "invalid match - regex"
42
+ end
43
+ end
44
+
36
45
  stack = []
37
46
  regex = ['']
38
47
  escape = false
@@ -143,17 +143,33 @@ module Fluent
143
143
  end
144
144
  end
145
145
 
146
+ # timekey should be unixtime as usual.
147
+ # So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
148
+ # We should check object_id stability to use object_id as optimization for comparing operations.
149
+ # e.g.)
150
+ # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
151
+ # => 1596187800
152
+ # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
153
+ # => true
154
+ def self.enable_optimize?
155
+ a1 = 2**30 - 1
156
+ a2 = 2**30 - 1
157
+ b1 = 2**62 - 1
158
+ b2 = 2**62 - 1
159
+ (a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
160
+ end
161
+
146
162
  # This is an optimization code. Current Struct's implementation is comparing all data.
147
163
  # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
148
164
  # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
149
165
  # This means that this class is one of the most called object in Fluentd.
150
166
  # See https://github.com/fluent/fluentd/pull/2560
151
- # But, this optimization has a side effect on Windows due to differing object_id.
167
+ # But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
152
168
  # This difference causes flood of buffer files.
153
- # So, this optimization should be enabled on non-Windows platform.
169
+ # So, this optimization should be enabled on `enable_optimize?` as true platforms.
154
170
  def hash
155
171
  timekey.object_id
156
- end unless Fluent.windows?
172
+ end if enable_optimize?
157
173
  end
158
174
 
159
175
  # for tests
@@ -604,7 +620,11 @@ module Fluent
604
620
 
605
621
  if chunk_size_over?(chunk)
606
622
  if format && empty_chunk
607
- log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
623
+ if chunk.bytesize > @chunk_limit_size
624
+ log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
625
+ else
626
+ log.warn "chunk size limit exceeds for an emitted event stream: #{chunk.size}records"
627
+ end
608
628
  end
609
629
  chunk.rollback
610
630
 
@@ -14,126 +14,5 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'json'
18
-
19
- require 'fluent/plugin/input'
20
- require 'fluent/config/error'
21
-
22
- module Fluent::Plugin
23
- class DummyInput < Input
24
- Fluent::Plugin.register_input('dummy', self)
25
-
26
- helpers :thread, :storage
27
-
28
- BIN_NUM = 10
29
- DEFAULT_STORAGE_TYPE = 'local'
30
-
31
- desc "The value is the tag assigned to the generated events."
32
- config_param :tag, :string
33
- desc "The number of events in event stream of each emits."
34
- config_param :size, :integer, default: 1
35
- desc "It configures how many events to generate per second."
36
- config_param :rate, :integer, default: 1
37
- desc "If specified, each generated event has an auto-incremented key field."
38
- config_param :auto_increment_key, :string, default: nil
39
- desc "The boolean to suspend-and-resume incremental value after restart"
40
- config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
41
- desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
42
- config_param :dummy, default: [{"message"=>"dummy"}] do |val|
43
- begin
44
- parsed = JSON.parse(val)
45
- rescue JSON::ParserError => ex
46
- # Fluent::ConfigParseError, "got incomplete JSON" will be raised
47
- # at literal_parser.rb with --use-v1-config, but I had to
48
- # take care at here for the case of --use-v0-config.
49
- raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
50
- end
51
- dummy = parsed.is_a?(Array) ? parsed : [parsed]
52
- dummy.each_with_index do |e, i|
53
- raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
54
- end
55
- dummy
56
- end
57
-
58
- def initialize
59
- super
60
- @storage = nil
61
- end
62
-
63
- def configure(conf)
64
- super
65
- @dummy_index = 0
66
- config = conf.elements.select{|e| e.name == 'storage' }.first
67
- @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
68
- end
69
-
70
- def multi_workers_ready?
71
- true
72
- end
73
-
74
- def start
75
- super
76
-
77
- @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
78
- @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
79
-
80
- if @auto_increment_key && !@storage.get(:auto_increment_value)
81
- @storage.put(:auto_increment_value, -1)
82
- end
83
-
84
- thread_create(:dummy_input, &method(:run))
85
- end
86
-
87
- def run
88
- batch_num = (@rate / BIN_NUM).to_i
89
- residual_num = (@rate % BIN_NUM)
90
- while thread_current_running?
91
- current_time = Time.now.to_i
92
- BIN_NUM.times do
93
- break unless (thread_current_running? && Time.now.to_i <= current_time)
94
- wait(0.1) { emit(batch_num) }
95
- end
96
- emit(residual_num) if thread_current_running?
97
- # wait for next second
98
- while thread_current_running? && Time.now.to_i <= current_time
99
- sleep 0.01
100
- end
101
- end
102
- end
103
-
104
- def emit(num)
105
- begin
106
- if @size > 1
107
- num.times do
108
- router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
109
- end
110
- else
111
- num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
112
- end
113
- rescue => _
114
- # ignore all errors not to stop emits by emit errors
115
- end
116
- end
117
-
118
- def generate
119
- d = @dummy[@dummy_index]
120
- unless d
121
- @dummy_index = 0
122
- d = @dummy[@dummy_index]
123
- end
124
- @dummy_index += 1
125
- if @auto_increment_key
126
- d = d.dup
127
- d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
128
- end
129
- d
130
- end
131
-
132
- def wait(time)
133
- start_time = Time.now
134
- yield
135
- sleep_time = time - (Time.now - start_time)
136
- sleep sleep_time if sleep_time > 0
137
- end
138
- end
139
- end
17
+ # Remove this file in fluentd v2
18
+ require_relative 'in_sample'
@@ -72,7 +72,7 @@ module Fluent::Plugin
72
72
  desc 'Add REMOTE_ADDR header to the record.'
73
73
  config_param :add_remote_addr, :bool, default: false
74
74
  config_param :blocking_timeout, :time, default: 0.5
75
- desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
75
+ desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
76
76
  config_param :cors_allow_origins, :array, default: nil
77
77
  desc 'Respond with empty gif image of 1x1 pixel.'
78
78
  config_param :respond_with_empty_img, :bool, default: false
@@ -490,7 +490,7 @@ module Fluent::Plugin
490
490
  # CORS check
491
491
  # ==========
492
492
  # For every incoming request, we check if we have some CORS
493
- # restrictions and white listed origins through @cors_allow_origins.
493
+ # restrictions and allow listed origins through @cors_allow_origins.
494
494
  unless @cors_allow_origins.nil?
495
495
  unless @cors_allow_origins.include?('*') or include_cors_allow_origin
496
496
  send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
@@ -0,0 +1,141 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'json'
18
+
19
+ require 'fluent/plugin/input'
20
+ require 'fluent/config/error'
21
+
22
+ module Fluent::Plugin
23
+ class SampleInput < Input
24
+ Fluent::Plugin.register_input('sample', self)
25
+ Fluent::Plugin.register_input('dummy', self)
26
+
27
+ helpers :thread, :storage
28
+
29
+ BIN_NUM = 10
30
+ DEFAULT_STORAGE_TYPE = 'local'
31
+
32
+ desc "The value is the tag assigned to the generated events."
33
+ config_param :tag, :string
34
+ desc "The number of events in event stream of each emits."
35
+ config_param :size, :integer, default: 1
36
+ desc "It configures how many events to generate per second."
37
+ config_param :rate, :integer, default: 1
38
+ desc "If specified, each generated event has an auto-incremented key field."
39
+ config_param :auto_increment_key, :string, default: nil
40
+ desc "The boolean to suspend-and-resume incremental value after restart"
41
+ config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
42
+ desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
43
+ config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
44
+ begin
45
+ parsed = JSON.parse(val)
46
+ rescue JSON::ParserError => ex
47
+ # Fluent::ConfigParseError, "got incomplete JSON" will be raised
48
+ # at literal_parser.rb with --use-v1-config, but I had to
49
+ # take care at here for the case of --use-v0-config.
50
+ raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
51
+ end
52
+ sample = parsed.is_a?(Array) ? parsed : [parsed]
53
+ sample.each_with_index do |e, i|
54
+ raise Fluent::ConfigError, "#{i}th element of sample, #{e}, is not a hash" unless e.is_a?(Hash)
55
+ end
56
+ sample
57
+ end
58
+
59
+ def initialize
60
+ super
61
+ @storage = nil
62
+ end
63
+
64
+ def configure(conf)
65
+ super
66
+ @sample_index = 0
67
+ config = conf.elements.select{|e| e.name == 'storage' }.first
68
+ @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
69
+ end
70
+
71
+ def multi_workers_ready?
72
+ true
73
+ end
74
+
75
+ def start
76
+ super
77
+
78
+ @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
79
+ # keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2
80
+ @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
81
+
82
+ if @auto_increment_key && !@storage.get(:auto_increment_value)
83
+ @storage.put(:auto_increment_value, -1)
84
+ end
85
+
86
+ thread_create(:sample_input, &method(:run))
87
+ end
88
+
89
+ def run
90
+ batch_num = (@rate / BIN_NUM).to_i
91
+ residual_num = (@rate % BIN_NUM)
92
+ while thread_current_running?
93
+ current_time = Time.now.to_i
94
+ BIN_NUM.times do
95
+ break unless (thread_current_running? && Time.now.to_i <= current_time)
96
+ wait(0.1) { emit(batch_num) }
97
+ end
98
+ emit(residual_num) if thread_current_running?
99
+ # wait for next second
100
+ while thread_current_running? && Time.now.to_i <= current_time
101
+ sleep 0.01
102
+ end
103
+ end
104
+ end
105
+
106
+ def emit(num)
107
+ begin
108
+ if @size > 1
109
+ num.times do
110
+ router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
111
+ end
112
+ else
113
+ num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
114
+ end
115
+ rescue => _
116
+ # ignore all errors not to stop emits by emit errors
117
+ end
118
+ end
119
+
120
+ def generate
121
+ d = @sample[@sample_index]
122
+ unless d
123
+ @sample_index = 0
124
+ d = @sample[@sample_index]
125
+ end
126
+ @sample_index += 1
127
+ if @auto_increment_key
128
+ d = d.dup
129
+ d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
130
+ end
131
+ d
132
+ end
133
+
134
+ def wait(time)
135
+ start_time = Time.now
136
+ yield
137
+ sleep_time = time - (Time.now - start_time)
138
+ sleep sleep_time if sleep_time > 0
139
+ end
140
+ end
141
+ end
@@ -27,13 +27,9 @@ module Fluent
27
27
  cert, key, extra = cert_option_server_validate!(conf)
28
28
 
29
29
  ctx = OpenSSL::SSL::SSLContext.new
30
- unless insecure
31
- # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
32
- # https://bugs.ruby-lang.org/issues/9424
33
- ctx.set_params({})
34
-
35
- ctx.ciphers = ciphers
36
- end
30
+ # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
31
+ # https://bugs.ruby-lang.org/issues/9424
32
+ ctx.set_params({}) unless insecure
37
33
 
38
34
  if conf.client_cert_auth
39
35
  ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
@@ -56,6 +52,7 @@ module Fluent
56
52
  end
57
53
 
58
54
  Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
55
+ ctx.ciphers = ciphers unless insecure
59
56
 
60
57
  ctx
61
58
  end
@@ -185,7 +182,7 @@ module Fluent
185
182
  list = []
186
183
  data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
187
184
  if list.length == 0
188
- log.warn "cert_path does not contain a valid certificate"
185
+ raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
189
186
  end
190
187
  list
191
188
  end
@@ -199,7 +199,7 @@ module Fluent
199
199
  list = []
200
200
  data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
201
201
  if list.length == 0
202
- log.warn "cert_path does not contain a valid certificate"
202
+ raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
203
203
  end
204
204
  list
205
205
  end
@@ -804,7 +804,7 @@ module Fluent
804
804
  Fluent::Engine.reload_config(conf)
805
805
  end
806
806
  rescue => e
807
- # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
807
+ # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
808
808
  # So need this rescue code
809
809
  $log.error("failed to reload config: #{e}")
810
810
  next
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.11.1'
19
+ VERSION = '1.11.2'
20
20
 
21
21
  end
@@ -1,33 +1,33 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/test/driver/input'
3
- require 'fluent/plugin/in_dummy'
3
+ require 'fluent/plugin/in_sample'
4
4
  require 'fileutils'
5
5
 
6
- class DummyTest < Test::Unit::TestCase
6
+ class SampleTest < Test::Unit::TestCase
7
7
  def setup
8
8
  Fluent::Test.setup
9
9
  end
10
10
 
11
11
  def create_driver(conf)
12
- Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
12
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::SampleInput).configure(conf)
13
13
  end
14
14
 
15
15
  sub_test_case 'configure' do
16
16
  test 'required parameters' do
17
17
  assert_raise_message("'tag' parameter is required") do
18
- Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
18
+ Fluent::Plugin::SampleInput.new.configure(config_element('ROOT',''))
19
19
  end
20
20
  end
21
21
 
22
22
  test 'tag' do
23
23
  d = create_driver(%[
24
- tag dummy
24
+ tag sample
25
25
  ])
26
- assert_equal "dummy", d.instance.tag
26
+ assert_equal "sample", d.instance.tag
27
27
  end
28
28
 
29
29
  config = %[
30
- tag dummy
30
+ tag sample
31
31
  ]
32
32
 
33
33
  test 'auto_increment_key' do
@@ -44,30 +44,30 @@ class DummyTest < Test::Unit::TestCase
44
44
  assert_equal 10, d.instance.rate
45
45
  end
46
46
 
47
- test 'dummy' do
47
+ test 'sample' do
48
48
  # hash is okay
49
- d = create_driver(config + %[dummy {"foo":"bar"}])
50
- assert_equal [{"foo"=>"bar"}], d.instance.dummy
49
+ d = create_driver(config + %[sample {"foo":"bar"}])
50
+ assert_equal [{"foo"=>"bar"}], d.instance.sample
51
51
 
52
52
  # array of hash is okay
53
- d = create_driver(config + %[dummy [{"foo":"bar"}]])
54
- assert_equal [{"foo"=>"bar"}], d.instance.dummy
53
+ d = create_driver(config + %[sample [{"foo":"bar"}]])
54
+ assert_equal [{"foo"=>"bar"}], d.instance.sample
55
55
 
56
56
  assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
57
- create_driver(config + %[dummy "foo"])
57
+ create_driver(config + %[sample "foo"])
58
58
  end
59
59
 
60
60
  assert_raise_message(/is not a hash/) do
61
- create_driver(config + %[dummy ["foo"]])
61
+ create_driver(config + %[sample ["foo"]])
62
62
  end
63
63
  end
64
64
  end
65
65
 
66
66
  sub_test_case "emit" do
67
67
  config = %[
68
- tag dummy
68
+ tag sample
69
69
  rate 10
70
- dummy {"foo":"bar"}
70
+ sample {"foo":"bar"}
71
71
  ]
72
72
 
73
73
  test 'simple' do
@@ -75,7 +75,7 @@ class DummyTest < Test::Unit::TestCase
75
75
  d.run(timeout: 0.5)
76
76
 
77
77
  d.events.each do |tag, time, record|
78
- assert_equal("dummy", tag)
78
+ assert_equal("sample", tag)
79
79
  assert_equal({"foo"=>"bar"}, record)
80
80
  assert(time.is_a?(Fluent::EventTime))
81
81
  end
@@ -86,20 +86,20 @@ class DummyTest < Test::Unit::TestCase
86
86
  d.run(timeout: 0.5)
87
87
 
88
88
  d.events.each_with_index do |(tag, _time, record), i|
89
- assert_equal("dummy", tag)
89
+ assert_equal("sample", tag)
90
90
  assert_equal({"foo"=>"bar", "id"=>i}, record)
91
91
  end
92
92
  end
93
93
  end
94
94
 
95
- TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
95
+ TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_sample', 'store' )
96
96
  FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
97
97
 
98
- sub_test_case 'when dummy plugin has storage which is not specified the path' do
98
+ sub_test_case 'when sample plugin has storage which is not specified the path' do
99
99
  config1 = {
100
- 'tag' => 'dummy',
100
+ 'tag' => 'sample',
101
101
  'rate' => '0',
102
- 'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
102
+ 'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
103
103
  'auto_increment_key' => 'id',
104
104
  }
105
105
  conf1 = config_element('ROOT', '', config1, [])
@@ -135,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
135
135
  end
136
136
  end
137
137
 
138
- sub_test_case 'when dummy plugin has storage which is specified the path' do
138
+ sub_test_case 'when sample plugin has storage which is specified the path' do
139
139
  setup do
140
140
  FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
141
141
  FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
@@ -144,9 +144,9 @@ class DummyTest < Test::Unit::TestCase
144
144
 
145
145
  config2 = {
146
146
  '@id' => 'test-02',
147
- 'tag' => 'dummy',
147
+ 'tag' => 'sample',
148
148
  'rate' => '0',
149
- 'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
149
+ 'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
150
150
  'auto_increment_key' => 'id',
151
151
  }
152
152
  conf2 = config_element('ROOT', '', config2, [
File without changes
@@ -15,4 +15,11 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
15
15
  certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/cert-with-CRLF.pem")
16
16
  assert_equal(1, certs.length)
17
17
  end
18
+
19
+ test 'raise an error for broken certificates_from_file file' do
20
+ d = Dummy.new
21
+ assert_raise Fluent::ConfigError do
22
+ certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
23
+ end
24
+ end
18
25
  end
@@ -1233,6 +1233,40 @@ class ServerPluginHelperTest < Test::Unit::TestCase
1233
1233
  waiting(10){ sleep 0.1 until received.bytesize == 8 }
1234
1234
  assert_equal "yay\nfoo\n", received
1235
1235
  end
1236
+
1237
+ test 'set ciphers' do
1238
+ cert_path = File.join(@server_cert_dir, "cert.pem")
1239
+ private_key_path = File.join(@certs_dir, "server.key.pem")
1240
+ create_server_pair_signed_by_self(cert_path, private_key_path, nil)
1241
+ tls_options = {
1242
+ protocol: :tls,
1243
+ version: :TLSv1_2,
1244
+ ciphers: 'SHA256',
1245
+ insecure: false,
1246
+ cert_path: cert_path,
1247
+ private_key_path: private_key_path,
1248
+ }
1249
+ conf = @d.server_create_transport_section_object(tls_options)
1250
+ ctx = @d.cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
1251
+ matched = false
1252
+ ctx.ciphers.each do |cipher|
1253
+ cipher_name, tls_version = cipher
1254
+ # OpenSSL 1.0.2: "TLSv1/SSLv3"
1255
+ # OpenSSL 1.1.1: "TLSv1.2"
1256
+ if tls_version == "TLSv1/SSLv3" || tls_version == "TLSv1.2"
1257
+ matched = true
1258
+ unless cipher_name.match(/#{conf.ciphers}/)
1259
+ matched = false
1260
+ break
1261
+ end
1262
+ end
1263
+ end
1264
+
1265
+ error_msg = build_message("Unexpected ciphers for #{conf.version}",
1266
+ "<?>\nwas expected to include only <?> ciphers for #{conf.version}",
1267
+ ctx.ciphers, conf.ciphers)
1268
+ assert(matched, error_msg)
1269
+ end
1236
1270
  end
1237
1271
  end
1238
1272
 
@@ -128,4 +128,12 @@ class SocketHelperTest < Test::Unit::TestCase
128
128
  client.close
129
129
  end
130
130
  end
131
+
132
+ test 'with empty cert file' do
133
+ cert_path = File.expand_path(File.dirname(__FILE__) + '/data/cert/empty.pem')
134
+
135
+ assert_raise Fluent::ConfigError do
136
+ SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, cert_path: cert_path)
137
+ end
138
+ end
131
139
  end
@@ -101,6 +101,17 @@ class MatchTest < Test::Unit::TestCase
101
101
  assert_or_not_match('a.b.** a.c', 'a.c.d')
102
102
  end
103
103
 
104
+ def test_regex_pattern
105
+ assert_glob_match('/a/', 'a')
106
+ assert_glob_not_match('/a/', 'abc')
107
+ assert_glob_match('/a.*/', 'abc')
108
+ assert_glob_not_match('/b.*/', 'abc')
109
+ assert_glob_match('/a\..*/', 'a.b.c')
110
+ assert_glob_not_match('/(?!a\.).*/', 'a.b.c')
111
+ assert_glob_not_match('/a\..*/', 'b.b.c')
112
+ assert_glob_match('/(?!a\.).*/', 'b.b.c')
113
+ end
114
+
104
115
  #def test_character_class
105
116
  # assert_match('[a]', 'a')
106
117
  # assert_match('[ab]', 'a')
@@ -6,7 +6,7 @@ require 'fluent/plugin/out_forward'
6
6
  require 'fluent/plugin/out_stdout'
7
7
  require 'fluent/plugin/out_exec'
8
8
  require 'fluent/plugin/in_forward'
9
- require 'fluent/plugin/in_dummy'
9
+ require 'fluent/plugin/in_sample'
10
10
  require 'fluent/plugin/filter_grep'
11
11
  require 'fluent/plugin/filter_stdout'
12
12
  require 'fluent/plugin/filter_parser'
@@ -74,7 +74,7 @@ class StaticConfigAnalysisTest < ::Test::Unit::TestCase
74
74
  c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
75
75
  ret = Fluent::StaticConfigAnalysis.call(c)
76
76
  assert_equal [Fluent::Plugin::ExecOutput, Fluent::Plugin::StdoutOutput, Fluent::Plugin::ForwardOutput], ret.outputs.map(&:plugin).map(&:class)
77
- assert_equal [Fluent::Plugin::DummyInput, Fluent::Plugin::ForwardInput], ret.inputs.map(&:plugin).map(&:class)
77
+ assert_equal [Fluent::Plugin::SampleInput, Fluent::Plugin::ForwardInput], ret.inputs.map(&:plugin).map(&:class)
78
78
  assert_equal [Fluent::Plugin::ParserFilter, Fluent::Plugin::StdoutFilter, Fluent::Plugin::GrepFilter], ret.filters.map(&:plugin).map(&:class)
79
79
  assert_equal 1, ret.labels.size
80
80
  assert_equal '@test', ret.labels[0].name
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.11.1
4
+ version: 1.11.2
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-22 00:00:00.000000000 Z
11
+ date: 2020-08-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -458,8 +458,6 @@ files:
458
458
  - example/copy_roundrobin.conf
459
459
  - example/counter.conf
460
460
  - example/filter_stdout.conf
461
- - example/in_dummy_blocks.conf
462
- - example/in_dummy_with_compression.conf
463
461
  - example/in_forward.conf
464
462
  - example/in_forward_client.conf
465
463
  - example/in_forward_shared_key.conf
@@ -468,6 +466,8 @@ files:
468
466
  - example/in_forward_workers.conf
469
467
  - example/in_http.conf
470
468
  - example/in_out_forward.conf
469
+ - example/in_sample_blocks.conf
470
+ - example/in_sample_with_compression.conf
471
471
  - example/in_syslog.conf
472
472
  - example/in_tail.conf
473
473
  - example/in_tcp.conf
@@ -605,6 +605,7 @@ files:
605
605
  - lib/fluent/plugin/in_http.rb
606
606
  - lib/fluent/plugin/in_monitor_agent.rb
607
607
  - lib/fluent/plugin/in_object_space.rb
608
+ - lib/fluent/plugin/in_sample.rb
608
609
  - lib/fluent/plugin/in_syslog.rb
609
610
  - lib/fluent/plugin/in_tail.rb
610
611
  - lib/fluent/plugin/in_tail/position_file.rb
@@ -815,13 +816,13 @@ files:
815
816
  - test/plugin/test_formatter_single_value.rb
816
817
  - test/plugin/test_formatter_tsv.rb
817
818
  - test/plugin/test_in_debug_agent.rb
818
- - test/plugin/test_in_dummy.rb
819
819
  - test/plugin/test_in_exec.rb
820
820
  - test/plugin/test_in_forward.rb
821
821
  - test/plugin/test_in_gc_stat.rb
822
822
  - test/plugin/test_in_http.rb
823
823
  - test/plugin/test_in_monitor_agent.rb
824
824
  - test/plugin/test_in_object_space.rb
825
+ - test/plugin/test_in_sample.rb
825
826
  - test/plugin/test_in_syslog.rb
826
827
  - test/plugin/test_in_tail.rb
827
828
  - test/plugin/test_in_tcp.rb
@@ -877,6 +878,7 @@ files:
877
878
  - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
878
879
  - test/plugin_helper/data/cert/cert_chains/cert-key.pem
879
880
  - test/plugin_helper/data/cert/cert_chains/cert.pem
881
+ - test/plugin_helper/data/cert/empty.pem
880
882
  - test/plugin_helper/data/cert/generate_cert.rb
881
883
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
882
884
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -1048,13 +1050,13 @@ test_files:
1048
1050
  - test/plugin/test_formatter_single_value.rb
1049
1051
  - test/plugin/test_formatter_tsv.rb
1050
1052
  - test/plugin/test_in_debug_agent.rb
1051
- - test/plugin/test_in_dummy.rb
1052
1053
  - test/plugin/test_in_exec.rb
1053
1054
  - test/plugin/test_in_forward.rb
1054
1055
  - test/plugin/test_in_gc_stat.rb
1055
1056
  - test/plugin/test_in_http.rb
1056
1057
  - test/plugin/test_in_monitor_agent.rb
1057
1058
  - test/plugin/test_in_object_space.rb
1059
+ - test/plugin/test_in_sample.rb
1058
1060
  - test/plugin/test_in_syslog.rb
1059
1061
  - test/plugin/test_in_tail.rb
1060
1062
  - test/plugin/test_in_tcp.rb
@@ -1110,6 +1112,7 @@ test_files:
1110
1112
  - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
1111
1113
  - test/plugin_helper/data/cert/cert_chains/cert-key.pem
1112
1114
  - test/plugin_helper/data/cert/cert_chains/cert.pem
1115
+ - test/plugin_helper/data/cert/empty.pem
1113
1116
  - test/plugin_helper/data/cert/generate_cert.rb
1114
1117
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
1115
1118
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem