fluentd 1.16.7-x86-mingw32 → 1.17.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +50 -0
  3. data/.github/DISCUSSION_TEMPLATE/q-a.yml +47 -0
  4. data/.github/workflows/test-ruby-head.yml +31 -0
  5. data/.github/workflows/test.yml +2 -9
  6. data/CHANGELOG.md +34 -42
  7. data/README.md +3 -1
  8. data/Rakefile +1 -1
  9. data/fluentd.gemspec +9 -9
  10. data/lib/fluent/command/binlog_reader.rb +1 -1
  11. data/lib/fluent/command/fluentd.rb +1 -1
  12. data/lib/fluent/config/configure_proxy.rb +2 -2
  13. data/lib/fluent/config/types.rb +1 -1
  14. data/lib/fluent/config/yaml_parser/parser.rb +0 -4
  15. data/lib/fluent/configurable.rb +2 -2
  16. data/lib/fluent/counter/mutex_hash.rb +1 -1
  17. data/lib/fluent/fluent_log_event_router.rb +0 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_http.rb +18 -53
  23. data/lib/fluent/plugin/in_tail.rb +34 -2
  24. data/lib/fluent/plugin/out_file.rb +0 -8
  25. data/lib/fluent/plugin/out_http.rb +125 -13
  26. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  27. data/lib/fluent/plugin/parser_json.rb +34 -9
  28. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  29. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  30. data/lib/fluent/registry.rb +6 -6
  31. data/lib/fluent/supervisor.rb +1 -1
  32. data/lib/fluent/test/output_test.rb +1 -1
  33. data/lib/fluent/unique_id.rb +1 -1
  34. data/lib/fluent/version.rb +1 -1
  35. data/lib/fluent/winsvc.rb +3 -28
  36. data/test/command/test_cat.rb +2 -2
  37. data/test/command/test_fluentd.rb +10 -57
  38. data/test/helper.rb +7 -27
  39. data/test/log/test_console_adapter.rb +10 -3
  40. data/test/plugin/data/log_numeric/01.log +0 -0
  41. data/test/plugin/data/log_numeric/02.log +0 -0
  42. data/test/plugin/data/log_numeric/12.log +0 -0
  43. data/test/plugin/data/log_numeric/14.log +0 -0
  44. data/test/plugin/in_tail/test_io_handler.rb +14 -13
  45. data/test/plugin/in_tail/test_position_file.rb +7 -6
  46. data/test/plugin/out_forward/test_ack_handler.rb +3 -3
  47. data/test/plugin/out_forward/test_socket_cache.rb +3 -3
  48. data/test/plugin/test_in_forward.rb +1 -2
  49. data/test/plugin/test_in_http.rb +24 -2
  50. data/test/plugin/test_in_monitor_agent.rb +6 -6
  51. data/test/plugin/test_in_syslog.rb +18 -25
  52. data/test/plugin/test_in_tail.rb +153 -4
  53. data/test/plugin/test_in_tcp.rb +1 -1
  54. data/test/plugin/test_in_udp.rb +10 -16
  55. data/test/plugin/test_out_exec_filter.rb +7 -12
  56. data/test/plugin/test_out_file.rb +2 -22
  57. data/test/plugin/test_out_forward.rb +3 -2
  58. data/test/plugin/test_out_http.rb +128 -0
  59. data/test/plugin/test_out_stream.rb +1 -1
  60. data/test/plugin/test_owned_by.rb +0 -1
  61. data/test/plugin/test_parser_json.rb +106 -31
  62. data/test/plugin/test_parser_msgpack.rb +127 -0
  63. data/test/plugin/test_storage.rb +0 -1
  64. data/test/plugin_helper/test_child_process.rb +4 -4
  65. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  66. data/test/plugin_helper/test_server.rb +41 -64
  67. data/test/plugin_helper/test_socket.rb +1 -1
  68. data/test/test_config.rb +0 -6
  69. data/test/test_event_router.rb +2 -2
  70. metadata +90 -21
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fcc3a30a9e314d87517f7be5560f5ff382d3335b0320be04ac6b3bb489bbdd8c
4
- data.tar.gz: e8a6e2ff32150bf564036724aa5e39ee45c933ea391498163a70b4aa3242473d
3
+ metadata.gz: cc7924012e09f9568cbda96bbb4d1449074a4db721795d74900261b984b04047
4
+ data.tar.gz: 8c3e8f5ef86237b5c18943ffbf393188338f9dbd3dbcac3d9fb321973ba3b43e
5
5
  SHA512:
6
- metadata.gz: d13313bad71d1bf23044f1a8fef3157c8e5392024cf6d2f5015fd4a2ab57eab0fc86b52fa9191705a82536cfafadafb7c19b975bc6c7d68f925ac132590a7278
7
- data.tar.gz: e0300f26f6713d5cada654b7774b0f0a25a4593cc2f961e484c54c5e15effdbdc87b17e2398f3f4430ca4e92e0a92e014223534e54cf44772a2c55918c73fdc6
6
+ metadata.gz: 7998a2f12349a767e7952dcf3a17a108cfb0f2e2063524862d97b83159012e8b753e46d2e4893e235e6a4bb1d43533f3068dc955601533fddf839f75edcca096
7
+ data.tar.gz: '09dac9716859404ddb3b563408cab32921c894300f2701b3e7aff89a608be154d0090e321adbbed9e0615459b06a4be3038f3e5b16206e624a0e1e770970c803'
@@ -0,0 +1,50 @@
1
+ title: "[QA (Japanese)]"
2
+ labels: ["Q&A (Japanese)"]
3
+ body:
4
+ - type: markdown
5
+ attributes:
6
+ value: |
7
+ 日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。
8
+ - type: textarea
9
+ id: question
10
+ attributes:
11
+ label: やりたいこと
12
+ description: |
13
+ 何について困っているのかを書いてください。試したことや実際の結果を示してください。
14
+ 期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。
15
+ render: markdown
16
+ validations:
17
+ required: true
18
+ - type: textarea
19
+ id: configuration
20
+ attributes:
21
+ label: 設定した内容
22
+ description: |
23
+ どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける)
24
+ render: apache
25
+ - type: textarea
26
+ id: logs
27
+ attributes:
28
+ label: ログの内容
29
+ description: |
30
+ Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける)
31
+ render: shell
32
+ - type: textarea
33
+ id: environment
34
+ attributes:
35
+ label: 環境について
36
+ description: |
37
+ - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
38
+ - Operating system: `cat /etc/os-release`
39
+ - Kernel version: `uname -r`
40
+
41
+ どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。
42
+ 必要な情報を記入することをおすすめします。
43
+ value: |
44
+ - Fluentd version:
45
+ - TD Agent version:
46
+ - Fluent Package version:
47
+ - Docker image (tag):
48
+ - Operating system:
49
+ - Kernel version:
50
+ render: markdown
@@ -0,0 +1,47 @@
1
+ title: "[Q&A]"
2
+ labels: ["Q&A"]
3
+ body:
4
+ - type: markdown
5
+ attributes:
6
+ value: |
7
+ It is recommended to support each other.
8
+ - type: textarea
9
+ id: question
10
+ attributes:
11
+ label: What is a problem?
12
+ description: |
13
+ A clear and concise description of what you want to happen.
14
+ What exactly did you do (or not do) that was effective (or ineffective)?
15
+ render: markdown
16
+ validations:
17
+ required: true
18
+ - type: textarea
19
+ id: configuration
20
+ attributes:
21
+ label: Describe the configuration of Fluentd
22
+ description: |
23
+ If there is the actual configuration of Fluentd, it will help.
24
+ - type: textarea
25
+ id: logs
26
+ attributes:
27
+ label: Describe the logs of Fluentd
28
+ description: |
29
+ If there are error logs of Fluentd, it will help.
30
+ - type: textarea
31
+ id: environment
32
+ attributes:
33
+ label: Environment
34
+ description: |
35
+ - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
36
+ - Operating system: `cat /etc/os-release`
37
+ - Kernel version: `uname -r`
38
+
39
+ Please describe your environment information. If will help to support.
40
+ value: |
41
+ - Fluentd version:
42
+ - TD Agent version:
43
+ - Fluent Package version:
44
+ - Docker image (tag):
45
+ - Operating system:
46
+ - Kernel version:
47
+ render: markdown
@@ -0,0 +1,31 @@
1
+ name: Test with Ruby head
2
+
3
+ on:
4
+ schedule:
5
+ - cron: '11 14 * * 0'
6
+ workflow_dispatch:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ${{ matrix.os }}
11
+ continue-on-error: true
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
16
+ ruby-version: ['head']
17
+
18
+ name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
19
+ steps:
20
+ - uses: actions/checkout@v3
21
+ - name: Set up Ruby
22
+ uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby-version }}
25
+ - name: Install addons
26
+ if: ${{ matrix.os == 'ubuntu-latest' }}
27
+ run: sudo apt-get install libgmp3-dev libcap-ng-dev
28
+ - name: Install dependencies
29
+ run: bundle install
30
+ - name: Run tests
31
+ run: bundle exec rake test TESTOPTS="-v --no-show-detail-immediately"
@@ -2,15 +2,9 @@ name: Test
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [v1.16]
6
- paths-ignore:
7
- - '*.md'
8
- - 'lib/fluent/version.rb'
5
+ branches: [master]
9
6
  pull_request:
10
- branches: [v1.16]
11
- paths-ignore:
12
- - '*.md'
13
- - 'lib/fluent/version.rb'
7
+ branches: [master]
14
8
 
15
9
  jobs:
16
10
  test:
@@ -29,7 +23,6 @@ jobs:
29
23
  uses: ruby/setup-ruby@v1
30
24
  with:
31
25
  ruby-version: ${{ matrix.ruby-version }}
32
- rubygems: latest
33
26
  - name: Install addons
34
27
  if: ${{ matrix.os == 'ubuntu-latest' }}
35
28
  run: sudo apt-get install libgmp3-dev libcap-ng-dev
data/CHANGELOG.md CHANGED
@@ -1,54 +1,46 @@
1
- # v1.16
2
-
3
- ## Release v1.16.7 - 2025/01/29
4
-
5
- ### Bug Fix
1
+ # v1.17
6
2
 
7
- * Windows: Fix `NoMethodError` of `--daemon` option
8
- https://github.com/fluent/fluentd/pull/4796
9
- * Windows: Fixed an issue where stopping the service immediately after startup could leave the processes
10
- https://github.com/fluent/fluentd/pull/4782
11
- * Windows: Fixed an issue where stopping service sometimes can not be completed forever
12
- https://github.com/fluent/fluentd/pull/4782
3
+ ## Release v1.17.0 - 2024/04/30
13
4
 
14
- ### Misc
5
+ ### Enhancement
15
6
 
16
- * Windows: Add workaround for unexpected exception
17
- https://github.com/fluent/fluentd/pull/4747
18
- * README: remove deprecated google analytics beacon
19
- https://github.com/fluent/fluentd/pull/4797
20
- * CI improvemnts
21
- https://github.com/fluent/fluentd/pull/4723
22
- https://github.com/fluent/fluentd/pull/4788
23
- https://github.com/fluent/fluentd/pull/4789
24
- https://github.com/fluent/fluentd/pull/4790
25
- https://github.com/fluent/fluentd/pull/4791
26
- https://github.com/fluent/fluentd/pull/4793
27
- https://github.com/fluent/fluentd/pull/4794
28
- https://github.com/fluent/fluentd/pull/4795
29
- https://github.com/fluent/fluentd/pull/4798
30
- https://github.com/fluent/fluentd/pull/4799
31
- https://github.com/fluent/fluentd/pull/4800
32
- https://github.com/fluent/fluentd/pull/4801
33
- https://github.com/fluent/fluentd/pull/4803
34
-
35
- ## Release v1.16.6 - 2024/08/16
7
+ * in_http: Recognize CSP reports as JSON data
8
+ https://github.com/fluent/fluentd/pull/4282
9
+ * out_http: Add option to reuse connections
10
+ https://github.com/fluent/fluentd/pull/4330
11
+ * in_tail: Expand glob capability for square brackets and one character matcher
12
+ https://github.com/fluent/fluentd/pull/4401
13
+ * out_http: Support AWS Signature Version 4 authentication
14
+ https://github.com/fluent/fluentd/pull/4459
36
15
 
37
16
  ### Bug Fix
38
17
 
39
- * YAML config syntax: Fix issue where `$log_level` element was not supported correctly
40
- https://github.com/fluent/fluentd/pull/4486
41
- * parser_json: Fix wrong LoadError warning
42
- https://github.com/fluent/fluentd/pull/4592
43
- * `fluentd` command: Fix `--plugin` (`-p`) option not to overwrite default value
44
- https://github.com/fluent/fluentd/pull/4605
18
+ * Make sure `parser_json` and `parser_msgpack` return `Hash`.
19
+ Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
20
+ https://github.com/fluent/fluentd/pull/4474
21
+ * filter_parser: Add error event for multiple parsed results
22
+ https://github.com/fluent/fluentd/pull/4478
45
23
 
46
24
  ### Misc
47
25
 
48
- * out_file: Add warn message for symlink_path setting
49
- https://github.com/fluent/fluentd/pull/4512
50
- * Keep console gem v1.23 to avoid LoadError
51
- https://github.com/fluent/fluentd/pull/4510
26
+ * Raise minimum required ruby version
27
+ https://github.com/fluent/fluentd/pull/4288
28
+ * Require missing dependent gems as of Ruby 3.4-dev
29
+ https://github.com/fluent/fluentd/pull/4411
30
+ * Minor code refactoring
31
+ https://github.com/fluent/fluentd/pull/4294
32
+ https://github.com/fluent/fluentd/pull/4299
33
+ https://github.com/fluent/fluentd/pull/4302
34
+ https://github.com/fluent/fluentd/pull/4320
35
+ * CI fixes
36
+ https://github.com/fluent/fluentd/pull/4369
37
+ https://github.com/fluent/fluentd/pull/4433
38
+ https://github.com/fluent/fluentd/pull/4452
39
+ https://github.com/fluent/fluentd/pull/4477
40
+ * github: unify YAML file extension to .yml
41
+ https://github.com/fluent/fluentd/pull/4429
42
+
43
+ # v1.16
52
44
 
53
45
  ## Release v1.16.5 - 2024/03/27
54
46
 
data/README.md CHANGED
@@ -29,7 +29,7 @@ Fluentd: Open-Source Log Collector
29
29
 
30
30
  ### Prerequisites
31
31
 
32
- - Ruby 2.4 or later
32
+ - Ruby 2.7 or later
33
33
  - git
34
34
 
35
35
  `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
@@ -71,3 +71,5 @@ See [SECURITY](SECURITY.md) to contact us about vulnerability.
71
71
  ## Contributors:
72
72
 
73
73
  Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
74
+
75
+ [<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)
data/Rakefile CHANGED
@@ -13,7 +13,7 @@ task test: [:base_test]
13
13
  namespace :build do
14
14
  desc 'Build gems for all platforms'
15
15
  task :all do
16
- Bundler.with_clean_env do
16
+ Bundler.with_original_env do
17
17
  %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
18
18
  ENV['GEM_BUILD_FAKE_PLATFORM'] = name
19
19
  Rake::Task["build"].execute
data/fluentd.gemspec CHANGED
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
16
16
  gem.require_paths = ["lib"]
17
17
  gem.license = "Apache-2.0"
18
18
 
19
- gem.required_ruby_version = '>= 2.4'
19
+ gem.required_ruby_version = '>= 2.7'
20
20
 
21
21
  gem.add_runtime_dependency("bundler")
22
22
  gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -29,12 +29,11 @@ Gem::Specification.new do |gem|
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
30
  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
31
31
  gem.add_runtime_dependency("webrick", ["~> 1.4"])
32
- gem.add_runtime_dependency("console", ["< 1.24"])
33
32
 
34
- # gems that aren't default gems as of Ruby 3.5
35
- # logger 1.6.3 or later cause bug on windows,
36
- # hold on 1.6.2 for a while. see https://github.com/ruby/logger/issues/107
37
- gem.add_runtime_dependency("logger", ["1.6.2"])
33
+ # gems that aren't default gems as of Ruby 3.4
34
+ gem.add_runtime_dependency("base64", ["~> 0.2"])
35
+ gem.add_runtime_dependency("csv", ["~> 3.2"])
36
+ gem.add_runtime_dependency("drb", ["~> 2.2"])
38
37
 
39
38
  # build gem for a certain platform. see also Rakefile
40
39
  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -51,12 +50,13 @@ Gem::Specification.new do |gem|
51
50
  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
52
51
  gem.add_development_dependency("simplecov", ["~> 0.7"])
53
52
  gem.add_development_dependency("rr", ["~> 3.0"])
54
- # timecop v0.9.9 supports `Process.clock_gettime`. It breaks some tests.
55
- # (https://github.com/fluent/fluentd/pull/4521)
56
- gem.add_development_dependency("timecop", ["< 0.9.9"])
53
+ gem.add_development_dependency("timecop", ["~> 0.9"])
57
54
  gem.add_development_dependency("test-unit", ["~> 3.3"])
58
55
  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
59
56
  gem.add_development_dependency("oj", [">= 2.14", "< 4"])
60
57
  gem.add_development_dependency("async", "~> 1.23")
61
58
  gem.add_development_dependency("async-http", ">= 0.50.0")
59
+ gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
60
+ gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
61
+ gem.add_development_dependency("rexml", ["~> 3.2"])
62
62
  end
@@ -130,7 +130,7 @@ module BinlogReaderCommand
130
130
  private
131
131
 
132
132
  def configure_option_parser
133
- @options.merge!(config_params: {})
133
+ @options[:config_params] = {}
134
134
 
135
135
  @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
136
136
 
@@ -46,7 +46,7 @@ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and
46
46
  }
47
47
 
48
48
  op.on('-p', '--plugin DIR', "add plugin directory") {|s|
49
- (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s
49
+ (cmd_opts[:plugin_dirs] ||= []) << s
50
50
  }
51
51
 
52
52
  op.on('-I PATH', "add library path") {|s|
@@ -201,7 +201,7 @@ module Fluent
201
201
 
202
202
  def overwrite_defaults(other) # other is owner plugin's corresponding proxy
203
203
  self.defaults = self.defaults.merge(other.defaults)
204
- self.sections.keys.each do |section_key|
204
+ self.sections.each_key do |section_key|
205
205
  if other.sections.has_key?(section_key)
206
206
  self.sections[section_key].overwrite_defaults(other.sections[section_key])
207
207
  end
@@ -274,7 +274,7 @@ module Fluent
274
274
  option_value_type!(name, opts, :deprecated, String)
275
275
  option_value_type!(name, opts, :obsoleted, String)
276
276
  if type == :enum
277
- if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
277
+ if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
278
278
  raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
279
279
  end
280
280
  end
@@ -123,7 +123,7 @@ module Fluent
123
123
 
124
124
  s = val.to_sym
125
125
  list = opts[:list]
126
- raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
126
+ raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
127
127
  unless list.include?(s)
128
128
  raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
129
129
  end
@@ -138,10 +138,6 @@ module Fluent
138
138
  sb.add_line('@id', v)
139
139
  end
140
140
 
141
- if (v = config.delete('$log_level'))
142
- sb.add_line('@log_level', v)
143
- end
144
-
145
141
  config.each do |key, val|
146
142
  if val.is_a?(Array)
147
143
  val.each do |v|
@@ -31,13 +31,13 @@ module Fluent
31
31
  super
32
32
  # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
33
33
  proxy = self.class.merged_configure_proxy
34
- proxy.params.keys.each do |name|
34
+ proxy.params.each_key do |name|
35
35
  next if name.to_s.start_with?('@')
36
36
  if proxy.defaults.has_key?(name)
37
37
  instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
38
38
  end
39
39
  end
40
- proxy.sections.keys.each do |name|
40
+ proxy.sections.each_key do |name|
41
41
  next if name.to_s.start_with?('@')
42
42
  subproxy = proxy.sections[name]
43
43
  if subproxy.multi?
@@ -54,7 +54,7 @@ module Fluent
54
54
  if mutex.try_lock
55
55
  locks[key] = mutex
56
56
  else
57
- locks.values.each(&:unlock)
57
+ locks.each_value(&:unlock)
58
58
  locks = {} # flush locked keys
59
59
  break
60
60
  end
@@ -47,8 +47,6 @@ module Fluent
47
47
  # it's not suppressed in default event router for non-log-event events
48
48
  log_event_router.suppress_missing_match!
49
49
 
50
- log_event_router = log_event_router
51
-
52
50
  unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
53
51
  unless unmatched_tags.empty?
54
52
  $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
@@ -56,7 +56,7 @@ module Fluent
56
56
 
57
57
  @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
58
58
 
59
- multi_workers_configured = owner.system_config.workers > 1 ? true : false
59
+ multi_workers_configured = owner.system_config.workers > 1
60
60
 
61
61
  using_plugin_root_dir = false
62
62
  unless @path
@@ -399,7 +399,7 @@ module Fluent
399
399
  end
400
400
 
401
401
  if chunk.slice(0, 2) == BUFFER_HEADER
402
- size = chunk.slice(2, 4).unpack('N').first
402
+ size = chunk.slice(2, 4).unpack1('N')
403
403
  if size
404
404
  return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
405
405
  end
@@ -238,17 +238,16 @@ module Fluent
238
238
  callback.call(file) if callback
239
239
  end
240
240
 
241
- URI_PARSER = URI::Parser.new
242
241
  ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
243
242
 
244
243
  def encode_key(metadata)
245
244
  k = @key ? metadata.variables[@key] : metadata.tag
246
245
  k ||= ''
247
- URI_PARSER.escape(k, ESCAPE_REGEXP)
246
+ URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
248
247
  end
249
248
 
250
249
  def decode_key(key)
251
- URI_PARSER.unescape(key)
250
+ URI::DEFAULT_PARSER.unescape(key)
252
251
  end
253
252
 
254
253
  def create_new_chunk(path, metadata, perm)
@@ -70,6 +70,13 @@ module Fluent::Plugin
70
70
  end
71
71
  end
72
72
  begin
73
+ # Note: https://github.com/fluent/fluentd/issues/4100
74
+ # If the parser returns multiple records from one raw_value,
75
+ # this returns only the first one record.
76
+ # This should be fixed in the future version.
77
+ result_time = nil
78
+ result_record = nil
79
+
73
80
  @parser.parse(raw_value) do |t, values|
74
81
  if values
75
82
  t = if @reserve_time
@@ -79,20 +86,31 @@ module Fluent::Plugin
79
86
  end
80
87
  @accessor.delete(record) if @remove_key_name_field
81
88
  r = handle_parsed(tag, record, t, values)
82
- return t, r
89
+
90
+ if result_record.nil?
91
+ result_time = t
92
+ result_record = r
93
+ else
94
+ if @emit_invalid_record_to_error
95
+ router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
96
+ "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
97
+ ))
98
+ end
99
+ end
83
100
  else
84
101
  if @emit_invalid_record_to_error
85
102
  router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
86
103
  end
87
- if @reserve_data
88
- t = time
89
- r = handle_parsed(tag, record, time, {})
90
- return t, r
91
- else
92
- return FAILED_RESULT
93
- end
104
+
105
+ next unless @reserve_data
106
+ next unless result_record.nil?
107
+
108
+ result_time = time
109
+ result_record = handle_parsed(tag, record, time, {})
94
110
  end
95
111
  end
112
+
113
+ return result_time, result_record
96
114
  rescue Fluent::Plugin::Parser::ParserError => e
97
115
  if @emit_invalid_record_to_error
98
116
  raise e
@@ -203,54 +203,24 @@ module Fluent::Plugin
203
203
  begin
204
204
  path = path_info[1..-1] # remove /
205
205
  tag = path.split('/').join('.')
206
- record_time, record = parse_params(params)
207
206
 
208
- # Skip nil record
209
- if record.nil?
210
- log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
211
- if @respond_with_empty_img
212
- return RESPONSE_IMG
213
- else
214
- if @use_204_response
215
- return RESPONSE_204
216
- else
217
- return RESPONSE_200
218
- end
207
+ mes = Fluent::MultiEventStream.new
208
+ parse_params(params) do |record_time, record|
209
+ if record.nil?
210
+ log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
211
+ next
219
212
  end
220
- end
221
213
 
222
- mes = nil
223
- # Support batched requests
224
- if record.is_a?(Array)
225
- mes = Fluent::MultiEventStream.new
226
- record.each do |single_record|
227
- add_params_to_record(single_record, params)
228
-
229
- if param_time = params['time']
230
- param_time = param_time.to_f
231
- single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
232
- elsif @custom_parser
233
- single_time = @custom_parser.parse_time(single_record)
234
- single_time, single_record = @custom_parser.convert_values(single_time, single_record)
235
- else
236
- single_time = convert_time_field(single_record)
237
- end
238
-
239
- mes.add(single_time, single_record)
240
- end
241
- else
242
214
  add_params_to_record(record, params)
243
215
 
244
216
  time = if param_time = params['time']
245
217
  param_time = param_time.to_f
246
218
  param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
247
219
  else
248
- if record_time.nil?
249
- convert_time_field(record)
250
- else
251
- record_time
252
- end
220
+ record_time.nil? ? convert_time_field(record) : record_time
253
221
  end
222
+
223
+ mes.add(time, record)
254
224
  end
255
225
  rescue => e
256
226
  if @dump_error_log
@@ -261,11 +231,7 @@ module Fluent::Plugin
261
231
 
262
232
  # TODO server error
263
233
  begin
264
- if mes
265
- router.emit_stream(tag, mes)
266
- else
267
- router.emit(tag, time, record)
268
- end
234
+ router.emit_stream(tag, mes) unless mes.empty?
269
235
  rescue => e
270
236
  if @dump_error_log
271
237
  log.error "failed to emit data", error: e
@@ -308,20 +274,18 @@ module Fluent::Plugin
308
274
  def parse_params_default(params)
309
275
  if msgpack = params['msgpack']
310
276
  @parser_msgpack.parse(msgpack) do |_time, record|
311
- return nil, record
277
+ yield nil, record
312
278
  end
313
279
  elsif js = params['json']
314
280
  @parser_json.parse(js) do |_time, record|
315
- return nil, record
281
+ yield nil, record
316
282
  end
317
283
  elsif ndjson = params['ndjson']
318
- events = []
319
284
  ndjson.split(/\r?\n/).each do |js|
320
285
  @parser_json.parse(js) do |_time, record|
321
- events.push(record)
286
+ yield nil, record
322
287
  end
323
288
  end
324
- return nil, events
325
289
  else
326
290
  raise "'json', 'ndjson' or 'msgpack' parameter is required"
327
291
  end
@@ -329,10 +293,9 @@ module Fluent::Plugin
329
293
 
330
294
  def parse_params_with_parser(params)
331
295
  if content = params[EVENT_RECORD_PARAMETER]
332
- @custom_parser.parse(content) { |time, record|
333
- raise "Received event is not #{@format_name}: #{content}" if record.nil?
334
- return time, record
335
- }
296
+ @custom_parser.parse(content) do |time, record|
297
+ yield time, record
298
+ end
336
299
  else
337
300
  raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
338
301
  end
@@ -573,6 +536,8 @@ module Fluent::Plugin
573
536
  params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
574
537
  elsif /^application\/json/.match?(@content_type)
575
538
  params['json'] = @body
539
+ elsif /^application\/csp-report/.match?(@content_type)
540
+ params['json'] = @body
576
541
  elsif /^application\/msgpack/.match?(@content_type)
577
542
  params['msgpack'] = @body
578
543
  elsif /^application\/x-ndjson/.match?(@content_type)
@@ -580,7 +545,7 @@ module Fluent::Plugin
580
545
  end
581
546
  path_info = uri.path
582
547
 
583
- if (@add_query_params)
548
+ if (@add_query_params)
584
549
 
585
550
  query_params = WEBrick::HTTPUtils.parse_query(uri.query)
586
551