fluentd 1.19.3 → 1.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9613f1dacdb60cfccb245e988e67cec002daee2f81cbe38c2b80c30d525ad7b8
-  data.tar.gz: 4a85aa6c3bdaebd9cd069d508ac969d8792586247bef1d73d25cc906f6cd7c87
+  metadata.gz: fa67a9199a9bcd73feb7dc286f4a1cb8347bd4d01d515561ceaf151671a9116e
+  data.tar.gz: ec7afdf8eea191d4badfb95361c3b41cfb35785c06dbacf3d0c0f65b89d734da
 SHA512:
-  metadata.gz: c2d63e0bd8cf51a61d254cfda559f38ed1bcc79ceb1bf1f0d0c80b21b922e67091d2ee8c291a0dc0fcac0110873d0045874df297e0795c0cd604af64f431f845
-  data.tar.gz: 5fe9e2450048bacc978389621b0e3f2c8aed0e9136ffe11192945a754173c1ce13ff50c310cfcdf1633d7dc784e14e9a4fc5a6de519da3bc81decb12f6d5b28d
+  metadata.gz: 1e900ecc699cb16836f2467105c09d51917446f7d7a885cbd66512e1fb8d24a2807a5b3ff7eeedec969e3e9dde7f8bc2cff40624e5c7d3d45d2bb5e4b536a75e
+  data.tar.gz: 8c258d373881b49dc90bc8e93ade34d46fb79b768203e05889a4ac3196361c97d185fa8b6c8006c9d985df65da20eabf1df62a802a3e365d27ba2f5cc345c1be

data/AGENTS.md

@@ -0,0 +1,46 @@
+# Guidance for coding agents
+
+This file is for AI coding agents (Claude Code, Cursor, Codex, Copilot, ...) working in this repository. Humans should read CONTRIBUTING.md.
+
+## Primary sources
+
+- Official documentation: https://docs.fluentd.org/
+  - Index for agents: https://docs.fluentd.org/llms.txt
+  - Every page is also available as Markdown by appending `.md` to its URL, e.g. https://docs.fluentd.org/configuration/config-file.md
+- Read the documentation before answering questions about configuration syntax or plugin parameters. Do not invent parameters; check the plugin's page or its `config_param` declarations in this repository.
+- Configuration has two formats.
+  - Classic: https://docs.fluentd.org/configuration/config-file.md
+  - YAML: https://docs.fluentd.org/configuration/config-file-yaml.md
+
+## Repository layout
+
+- `lib/fluent/plugin/` – bundled plugins (`in_*.rb`, `out_*.rb`, `filter_*.rb`, `parser_*.rb`, `formatter_*.rb`, `buf_*.rb`, `storage_*.rb`, `sd_*.rb`, `metrics_*.rb`)
+- `lib/fluent/plugin_helper/` – plugin helpers
+- `lib/fluent/config/` – configuration parsers (classic and YAML)
+- `test/` – tests written with test-unit
+- `templates/new_gem/` – files generated by `fluent-plugin-generate`
+
+## Developing
+
+- Install: `bundle install`
+- Run all tests: `bundle exec rake test`
+- Run one file: `bundle exec ruby -Ilib -Itest test/plugin/test_in_tail.rb`
+- Plugin API reference: https://docs.fluentd.org/plugin-development/README.md
+- Plugin helper reference: https://docs.fluentd.org/plugin-helper-overview/README.md
+- Test helpers for plugins: https://docs.fluentd.org/plugin-development/plugin-test-code.md
+
+## Conventions
+
+- Follow CONTRIBUTING.md. Commits need a DCO sign-off (`git commit -s`).
+- Keep changes minimal and focused; do not reformat unrelated code.
+- Add or update tests under `test/` for behavior changes.
+- Do not add runtime dependencies without discussion.
+- Match the surrounding code style; there is no autoformatter to run. CI runs `rubocop`, but only the Security and Performance cops are enabled.
+- Fluentd runs on Linux, macOS and Windows. CI tests `ubuntu-latest`, `macos-latest`, `windows-latest` and `windows-11-arm`. Assume your change must work on Windows.
+
+## Issues and pull requests
+
+- Pull requests follow `.github/PULL_REQUEST_TEMPLATE.md`. Keep its headings, including `Docs Changes` and `Release Note`, and write `Fixes #<n>` when an issue exists.
+- Blank issues are disabled. Use `.github/ISSUE_TEMPLATE/bug_report.yml` or `.github/ISSUE_TEMPLATE/feature_request.yml` and answer every required field.
+- Questions belong in [Discussions](https://github.com/fluent/fluentd/discussions), not in issues.
+- Note that `gh pr create --body` and `gh issue create --body` skip the template. Pass `--template`, or reproduce the template's sections in the body you supply.

data/CHANGELOG.md

@@ -1,5 +1,125 @@
+# v1.20
+
+## Release v1.20.0 - 2026/10/23
+
+### Enhancement
+
+* Replace Yajl with built-in JSON for simple cases https://github.com/fluent/fluentd/pull/5090
+* out_forward: add `send_keepalive_packet` option to enable TCP keepalive (SO_KEEPALIVE) https://github.com/fluent/fluentd/pull/5262
+* in_exec: add `command_timeout` option to limit child process execution time https://github.com/fluent/fluentd/pull/5320
+* Replace Yajl with JSON for handling JSON stream https://github.com/fluent/fluentd/pull/5383
+  * When an input stream ends in the middle of a JSON document, Fluentd now logs a warning and discards the incomplete tail instead of raising `ParseError`.
+* system_config: add `umask` option to `<system>` directive https://github.com/fluent/fluentd/pull/5390
+* parser_syslog: accept a space after RFC3164 priority https://github.com/fluent/fluentd/pull/5449
+* in_tail: add opt-in `enable_dir_watcher` and `dir_watcher_limit` to discover new files earlier by watching directories https://github.com/fluent/fluentd/pull/5528
+* in_tail: add opt-in worker-thread parsing via `num_threads` to offload parsing from the event loop https://github.com/fluent/fluentd/pull/5529
+  * The default `num_threads 1` keeps the previous synchronous behavior.
+  * In worker mode, `monitor_agent` also reports `worker_pending_batch_count`, `worker_pending_bytes` and `worker_parse_error_count`.
+* buffer: add `evacuate_limit_size` to keep evacuated chunks from filling the disk https://github.com/fluent/fluentd/pull/5473
+  * This option sets an approximate limit on the total size of the chunk files evacuated when the retry limit is reached, including the ones evacuated before. Chunks that do not fit are purged without being evacuated.
+  * The default `nil` keeps the previous behavior. Set `0` to disable evacuation.
+* in_forward: accept metadata on encapsulated timestamps sent by Fluent Bit v5.0.5 or later https://github.com/fluent/fluentd/pull/5411
+* Enable Ruby JIT by default https://github.com/fluent/fluentd/pull/5464
+  * The system configuration `enable_jit` now defaults to `true`.
+  * JIT increases memory usage of each worker process. Set `enable_jit false` in the `<system>` section to restore the previous behavior.
+  * This has no effect on Windows because YJIT is not supported there.
+* fluent-plugin-generate: add .gitignore file https://github.com/fluent/fluentd/pull/5370
+* Performance improvements
+  * https://github.com/fluent/fluentd/pull/5281
+  * https://github.com/fluent/fluentd/pull/5288
+  * https://github.com/fluent/fluentd/pull/5289
+  * https://github.com/fluent/fluentd/pull/5344
+  * https://github.com/fluent/fluentd/pull/5353
+  * https://github.com/fluent/fluentd/pull/5354
+  * https://github.com/fluent/fluentd/pull/5357
+  * https://github.com/fluent/fluentd/pull/5287
+  * https://github.com/fluent/fluentd/pull/5278
+  * https://github.com/fluent/fluentd/pull/5567
+
+### Bug Fix
+
+* http_server helper: Fix IPv6 bind address support in URI construction https://github.com/fluent/fluentd/pull/5239
+  * Fixed `URI::InvalidURIError` when binding to IPv6 addresses (e.g., `::`, `::1`)
+  * IPv6 addresses are now properly bracketed in URIs per RFC 3986 (e.g., `http://[::]:24231`)
+  * Handles pre-bracketed addresses correctly to avoid double-bracketing
+  * Affects all plugins using http_server helper with IPv6 bind addresses
+* out_file: create symlinks for all chunks sharing the latest timekey https://github.com/fluent/fluentd/pull/5476
+* time: fix TimeFormatter returning nil for the Unix epoch https://github.com/fluent/fluentd/pull/5420
+* in_tail: fix log duplication when a rotated file is appended during `rotate_wait` with `follow_inodes` https://github.com/fluent/fluentd/pull/5499
+* in_tail: fix `NameError` raised when flushing buffered lines with `emit_unmatched_lines true` https://github.com/fluent/fluentd/pull/5513
+* in_tail: fix unread lines never being collected when a throttled file is removed from the watch list https://github.com/fluent/fluentd/pull/5515
+* in_tail: fix shutdown errors and timeouts caused by an ongoing watcher refresh https://github.com/fluent/fluentd/pull/5530
+* in_tail: fix closed stream error when file rotation overlaps with shutdown https://github.com/fluent/fluentd/pull/5551
+* in_tail: use a monotonic clock for multiline flush timers so that they are not affected by wall-clock adjustments https://github.com/fluent/fluentd/pull/5542
+* buffer: fix a race that could make the buffer `stage_size` gauge transiently negative while a write and a chunk enqueue overlap https://github.com/fluent/fluentd/pull/5481
+* buffer: fix an enqueued unstaged file chunk being re-staged and flushed twice, which caused `closed stream` / `ENOENT` errors and leaked buffer size gauges https://github.com/fluent/fluentd/pull/5517
+* parser_msgpack: fix leftover data from one parse call leaking into the next https://github.com/fluent/fluentd/pull/5553
+* http_server helper: fix HTTP endpoints never recovering after a bind failure https://github.com/fluent/fluentd/pull/5523
+  * The plugin now fails to start, and the worker is restarted until the port is available.
+* root_agent: fix `emit_records` and `emit_size` never counted for inputs with `@label` https://github.com/fluent/fluentd/pull/5526
+* supervisor: fix graceful config reload failing for YAML configuration https://github.com/fluent/fluentd/pull/5522
+  * It also fixes `<<: !include` of a file that contains a mapping.
+
+### Misc
+
+* Drop Ruby 3.2 support https://github.com/fluent/fluentd/pull/5465
+* out_http: remove the Net::HTTP extra_chain_cert monkey patch https://github.com/fluent/fluentd/pull/5484
+* gem: add aarch64-mingw-ucrt gem https://github.com/fluent/fluentd/pull/5400
+* gem: add json gem (>= 2.21) as runtime dependency and remove Refinements workaround https://github.com/fluent/fluentd/pull/5429
+* in_tail: internal refactoring
+  * https://github.com/fluent/fluentd/pull/5518
+  * https://github.com/fluent/fluentd/pull/5520
+  * https://github.com/fluent/fluentd/pull/5521
+  * https://github.com/fluent/fluentd/pull/5527
+  * https://github.com/fluent/fluentd/pull/5554
+
 # v1.19
 
+## Release v1.19.4 - 2026/09/28
+
+### Bug Fix
+
+* buffer: enforce decompression_size_limit on the chunk IO path https://github.com/fluent/fluentd/pull/5504
+* buffer: clamp exported buffer size metrics to non-negative values https://github.com/fluent/fluentd/pull/5488
+* buffer: fix spurious `BufferOverflowError` caused by `queue_size` leaking when a chunk purge fails https://github.com/fluent/fluentd/pull/5487
+* buffer: fix `stage_byte_size` leak on staged to unstaged chunk demotion that could eventually raise spurious `BufferOverflowError` in plugins implementing `#format` https://github.com/fluent/fluentd/pull/5456
+* in_syslog: enforce message_length_limit on TCP/TLS transport https://github.com/fluent/fluentd/pull/5502
+  * The default value of `message_length_limit` is changed from 2048 to 8192 to match rsyslog's default `MaxMessageSize`.
+* output: fix incomplete path traversal check in extract_placeholders https://github.com/fluent/fluentd/pull/5501
+* output: treat JSON::GeneratorError as unrecoverable error https://github.com/fluent/fluentd/pull/5423
+* parser_syslog: fix NameError when RFC5424 timestamp has repeated spaces https://github.com/fluent/fluentd/pull/5500
+* parser_syslog: fix NameError when RFC3164 timestamp has repeated spaces https://github.com/fluent/fluentd/pull/5497
+* parser_syslog: avoid excessive backtracking when parsing malformed RFC5424 structured data https://github.com/fluent/fluentd/pull/5444
+* chunk: ensure to close the Tempfile for decompressed data https://github.com/fluent/fluentd/pull/5486
+* plugin base: bound the number of worker lock files by hashing the path into a fixed set of buckets https://github.com/fluent/fluentd/pull/5471
+* out_forward: stop the endless "ack in response and chunk id in sent data are different" warning storm by discarding (instead of reusing) a keepalive socket whose ack failed or came back with a mismatched chunk id https://github.com/fluent/fluentd/pull/5445
+* config: accept empty lines in quoted strings https://github.com/fluent/fluentd/pull/5478
+* config: fix a config error when a single scalar value is given to an array option in YAML config syntax (for example `retryable_response_codes: 503`) https://github.com/fluent/fluentd/pull/5433
+* supervisor: reduce memory usage of cleanup_lock_dir with huge number of lock files https://github.com/fluent/fluentd/pull/5472
+
+### Enhancement
+
+* in_http: add `<auth>` for basic authentication and `<security>` for client network allowlisting https://github.com/fluent/fluentd/pull/5503
+
+### Misc
+
+* gem: support json gem v3.x https://github.com/fluent/fluentd/pull/5493
+* Add `allow_comments: true` option for json parser https://github.com/fluent/fluentd/pull/5432
+* Set `allow_duplicate_key: true` for all `JSON.parse` call https://github.com/fluent/fluentd/pull/5430
+* CI fixes
+  * https://github.com/fluent/fluentd/pull/5495
+  * https://github.com/fluent/fluentd/pull/5477
+  * https://github.com/fluent/fluentd/pull/5466
+  * https://github.com/fluent/fluentd/pull/5460
+  * https://github.com/fluent/fluentd/pull/5455
+  * https://github.com/fluent/fluentd/pull/5454
+  * https://github.com/fluent/fluentd/pull/5453
+  * https://github.com/fluent/fluentd/pull/5452
+  * https://github.com/fluent/fluentd/pull/5434
+  * https://github.com/fluent/fluentd/pull/5426
+  * https://github.com/fluent/fluentd/pull/5425
+  * https://github.com/fluent/fluentd/pull/5424
+
 ## Release v1.19.3 - 2026/06/25
 
 ### Bug Fix

data/GithubWorkflow.md

@@ -9,7 +9,7 @@ Table of Contents
 * [Create a Pull Request](#create-a-pull-request)
 
 
-The [fluentd](https://github.com/fluent/fluentd.git) code is hosted on Github (https://github.com/fluent/fluentd). The repository is called `upstream`. Contributors will develop and commit their changes in a clone of upstream repository. Then contributors push their change to their forked repository (`origin`) and create a Pull Request (PR), the PR will be merged to `upstream` repository if it meets the all the necessary requirements.		
+The [fluentd](https://github.com/fluent/fluentd.git) code is hosted on Github (https://github.com/fluent/fluentd). The repository is called `upstream`. Contributors will develop and commit their changes in a clone of upstream repository. Then contributors push their change to their forked repository (`origin`) and create a Pull Request (PR), the PR will be merged to `upstream` repository if it meets all the necessary requirements.		
 
 ## Fork a repository
 

data/MAINTAINERS.md

@@ -7,10 +7,10 @@
 - [Satoshi Tagomori](https://github.com/tagomoris)
 - [Toru Takahashi](https://github.com/toru-takahashi), [Treasure Data](https://www.treasuredata.com/)
 - [Eduardo Silva](https://github.com/edsiper), [Chronosphere](https://chronosphere.io/)
-- [Fujimoto Seiji](https://github.com/fujimots)
+- [Fujimoto Seiji](https://github.com/fujimotos)
 - [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
 - [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)
-- [Daijiro Fukuda](https://github.com/daipom), [ClearCode](https://www.clear-code.com/)
+- [Daijiro Fukuda](https://github.com/daipom)
 
 ## Sub Projects: Fluent Bit
 

data/README.md

@@ -5,6 +5,7 @@ Fluentd: Open-Source Log Collector
 [![Test with Ruby head](https://github.com/fluent/fluentd/actions/workflows/test-ruby-head.yml/badge.svg)](https://github.com/fluent/fluentd/actions/workflows/test-ruby-head.yml)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/fluent/fluentd/badge)](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)
+[![Backport Pull Requests](https://github.com/fluent/fluentd/actions/workflows/backport.yml/badge.svg)](https://github.com/fluent/fluentd/actions/workflows/backport.yml)
 
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
@@ -24,11 +25,11 @@ Fluentd: Open-Source Log Collector
 ### Branch
 
 - master: For v1 development.
-- v0.12: For v0.12. This is deprecated version. we already stopped supporting (See https://www.fluentd.org/blog/drop-schedule-announcement-in-2019).
+- v0.12: For v0.12. This is a deprecated version. we already stopped supporting (See https://www.fluentd.org/blog/drop-schedule-announcement-in-2019).
 
 ### Prerequisites
 
-- Ruby 3.2 or later
+- Ruby 3.3 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.

data/Rakefile

@@ -6,6 +6,7 @@ require 'rake/testtask'
 require 'rake/clean'
 
 require_relative 'tasks/benchmark'
+require_relative 'tasks/backport'
 
 task test: [:base_test]
 
@@ -16,7 +17,7 @@ namespace :build do
   desc 'Build gems for all platforms'
   task :all do
     Bundler.with_original_env do
-      %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
+      %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt aarch64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute
       end

data/SECURITY.md

@@ -4,10 +4,11 @@
 
 |  Version  |     Supported      |
 |-----------|--------------------|
+| 1.20.x    | :white_check_mark: |
 | 1.19.x    | :white_check_mark: |
 | 1.18.x    | :x:                |
 | 1.17.x    | :x:                |
-| 1.16.x    | :white_check_mark: |
+| 1.16.x    | :x:                |
 | <= 1.15.x | :x:                |
 
 ## Reporting a Vulnerability
@@ -15,8 +16,32 @@
 Please post your vulnerability report from the following page:
 https://github.com/fluent/fluentd/security/advisories
 
+> [!CAUTION]
+> In above contact form, we accept about **ONLY** Fluentd's vulnerability, in contrast, **REJECT** your vulnerability report about Fluentd derivative products such as fluent-package, Fluentd container images and Fluentd kubernetes deamonset images and so on. There are appropriate contact forms for every these derivative products. See the following notice.
+
 > [!NOTE]
 > If you use fluent-package, please check [fluent-package-builder](https://github.com/fluent/fluent-package-builder/blob/master/SECURITY.md) and report it there.
 
 > [!NOTE]
 > If you use a Docker image of Fluentd, please check [Fluentd Docker Image](https://github.com/fluent/fluentd-docker-image/blob/master/SECURITY.md) and report it there.
+
+
+## Out of Scope / Non-Vulnerabilities
+
+Before reporting a vulnerability, please check if it falls under the following expected behaviors, design choices, or configuration issues. Reports regarding these items will be closed immediately as **not a bug** or **out of scope**.
+
+### 1. `in_debug_agent` and arbitrary code execution (`eval`)
+
+* **Description:** The `in_debug_agent` plugin is a developer tool designed specifically for debugging a running Fluentd process via dRuby. It requires explicit activation in the local configuration file and is disabled by default.
+* **Why it is not a vulnerability:** The ability to execute arbitrary Ruby code (`eval`) is **by design**. It functions similarly to `gdbserver`, allowing maintainers to inspect internal process states. It does not introduce a security risk unless an administrator explicitly enables it and exposes the port to an untrusted network, which constitutes a severe misconfiguration rather than a flaw in Fluentd itself.
+
+### 2. `record_transformer` with `enable_ruby`
+
+* **Description:** The `enable_ruby` option in the `record_transformer` plugin allows administrators to use Ruby expressions within the configuration file to transform log records dynamically.
+* **Why it is not a vulnerability:** Executing the Ruby code block defined in the configuration file is **by design**. It does not allow arbitrary code execution from untrusted log inputs unless the user explicitly writes insecure expressions (e.g., passing untrusted input directly into another dynamic evaluation nested inside the block). If an attacker can modify the local Fluentd configuration file to enable this feature and inject malicious Ruby code, the system is already fully compromised at the OS/infrastructure level (Post-Exploitation). We do not accept RCE reports that rely on the ability to modify the local configuration file.
+
+### 3. ReDoS (Regular Expression Denial of Service) reproducible only on Ruby 3.2 or older
+
+* **Description:** Potential algorithmic complexity issues in regular expressions that could lead to High CPU usage or Denial of Service on older Ruby runtimes.
+* **Why it is out of scope:** Fluentd officially supports modern Ruby environments. Our primary distribution channels including the official `fluent-package` and official Docker images ship with **Ruby 3.3 or later (e.g., Ruby 3.4)**, which includes built-in, robust mitigation against ReDoS at the language level (such as matching timeouts and engine-level optimizations). We do not accept ReDoS reports that only impact legacy Ruby versions (Ruby 3.2 or older) unless the issue can be demonstrated to cause a practical denial of service on Ruby 3.3+.  
+  However, if you find inefficient regular expressions, we welcome them as standard bug reports or performance improvement suggestions via regular GitHub Issues rather than security advisories.

data/lib/fluent/command/cat.rb

@@ -107,7 +107,7 @@ rescue
 end
 
 require 'socket'
-require 'yajl'
+require 'json'
 require 'msgpack'
 require 'fluent/ext_monitor_require'
 
@@ -304,7 +304,7 @@ class Writer
   end
 
   def abort_message(time, record)
-    $stdout.puts "!#{time}:#{Yajl.dump(record)}"
+    $stdout.puts "!#{time}:#{JSON.generate(record)}"
   end
 end
 
@@ -326,7 +326,7 @@ case format
 when 'json'
   begin
     while line = $stdin.gets
-      record = Yajl.load(line)
+      record = JSON.parse(line, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       w.write(record)
     end
   rescue

data/lib/fluent/compat/exec_util.rb

@@ -15,8 +15,9 @@
 #
 
 require 'msgpack'
-require 'yajl'
+require 'json'
 
+require 'fluent/env'
 require 'fluent/engine'
 require 'fluent/plugin'
 require 'fluent/parser'
@@ -76,10 +77,24 @@ module Fluent
       end
 
       class JSONParser < Parser
+        BYTES_TO_READ = 8192
+
         def call(io)
-          y = Yajl::Parser.new
-          y.on_parse_complete = @on_message
-          y.parse(io)
+          parser = JSON::ResumableParser.new(**Fluent::DEFAULT_JSON_PARSE_OPTIONS)
+          begin
+            chunk = +"".b
+            while io.readpartial(BYTES_TO_READ, chunk)
+              parser << chunk
+              while parser.parse
+                @on_message.call(parser.value)
+              end
+            end
+          rescue EOFError
+            unless parser.empty?
+              $log&.warn "JSON stream ended in the middle of a document; " \
+                         "discarding incomplete data"
+            end
+          end
         end
       end
 
@@ -115,7 +130,7 @@ module Fluent
 
       class JSONFormatter < Formatter
         def call(record, out)
-          out << Yajl.dump(record) << "\n"
+          out << JSON.generate(record) << "\n"
         end
       end
 

data/lib/fluent/config/basic_parser.rb

@@ -28,7 +28,6 @@ module Fluent
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
       ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
-      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/
 
       module ClassMethods
         def symbol(string)

data/lib/fluent/config/literal_parser.rb

@@ -17,15 +17,24 @@
 require 'stringio'
 
 require 'json'
-require 'yajl'
 require 'socket'
 require 'ripper'
 
+require 'fluent/env'
 require 'fluent/config/basic_parser'
 
 module Fluent
   module Config
     class LiteralParser < BasicParser
+      # A physical line break (LF, CR, or CRLF) inside a quoted string.
+      # CRLF is normalized to LF so that the same config text does not produce a
+      # different value depending on whether the file was saved with LF or CRLF.
+      # A lone CR is kept as-is, and an escaped "\r\n" still produces CRLF.
+      LINE_BREAK = /\r\n|[\r\n]/
+      # A backslash immediately followed by a physical line break.
+      # It works as a line continuation, so both are stripped from the value.
+      LINE_CONTINUATION = /\\#{LINE_BREAK}/o
+
       def self.unescape_char(c)
         case c
         when '"'
@@ -98,11 +107,10 @@ module Fluent
             else
               return string.join
             end
-          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
-            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
-              string << s
-            end
-            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
+          elsif skip(LINE_CONTINUATION)
+            next
+          elsif s = scan(LINE_BREAK)
+            string << (s == "\r\n" ? "\n" : s)
           elsif s = scan(/\\./)
             string << eval_escape_char(s[1,1])
           elsif skip(/\#\{/)
@@ -125,6 +133,8 @@ module Fluent
             string << "'"
           elsif s = scan(/\\\\/)
             string << "\\"
+          elsif s = scan(LINE_BREAK)
+            string << (s == "\r\n" ? "\n" : s)
           elsif s = scan(/./)
             string << s
           else
@@ -225,8 +235,10 @@ EOM
 
       def scan_json(is_array)
         result = nil
-        # Yajl does not raise ParseError for incomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
-        # This is the reason to use JSON module.
+        # scan_json detects the end of a JSON literal by repeatedly attempting
+        # to parse the buffer and treating a successful parse as completion.
+        # This requires a parser that raises JSON::ParserError for incomplete
+        # input like '[1', '{"h"', '{"h":' or '{"h1":1'.
 
         buffer = (is_array ? "[" : "{")
         line_buffer = ""
@@ -241,7 +253,7 @@ EOM
             # '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
             parsed = nil
             begin
-              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
+              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"), **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
             rescue JSON::ParserError
               # This '#' is in json string literals
             end
@@ -276,7 +288,7 @@ EOM
 
           line_buffer << char
           begin
-            result = JSON.parse(buffer + line_buffer)
+            result = JSON.parse(buffer + line_buffer, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
           rescue JSON::ParserError
             # Incomplete json string yet
           end

data/lib/fluent/config/types.rb

@@ -201,7 +201,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -228,7 +228,15 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) : val.strip.split(/\s*,\s*/)
+              elsif val.is_a?(Array)
+                val
+              elsif val.is_a?(Numeric) || val == true || val == false
+                # Wrap only the bare scalars that Psych/YAML actually produces
+                # here (nil is handled by the early return above). Any other
+                # type (Hash, Symbol, Time, arbitrary objects) falls through and
+                # still raises "array required" below.
+                [val]
               else
                 val
               end

data/lib/fluent/config/yaml_parser/loader.rb

@@ -71,9 +71,17 @@ module Fluent
           else
             pattern = path
           end
+          # A single file keeps its own shape; only glob matches are aggregated.
+          return load(pattern) if File.file?(pattern)
+
           result = []
           Dir.glob(pattern).sort.each do |path|
-            result.concat(load(Pathname.new(path)))
+            loaded = load(Pathname.new(path))
+            if loaded.is_a?(Hash)
+              result.push(loaded)
+            else
+              result.concat(loaded)
+            end
           end
           result
         rescue SystemCallError => e
@@ -93,6 +101,9 @@ module Fluent
 
           def revive_hash(hash, o)
             super(hash, o).tap do |r|
+              if r[SHOVEL].is_a?(Array) && r[SHOVEL].all?(Hash)
+                r[SHOVEL] = r[SHOVEL].reverse_each.reduce({}) { |a, e| a.merge!(e) }
+              end
               if r[SHOVEL].is_a?(Hash)
                 h2 = {}
                 r.each do |k, v|

data/lib/fluent/config/yaml_parser/parser.rb

@@ -106,19 +106,20 @@ module Fluent
 
         def included_sections_build(config, section_builder, indent: 0)
           config.each_entry do |e|
-            k = e.keys.first
-            cc = e.delete(k)
-            case k
-            when 'label'
-              section_builder.add_section(label_build(cc, indent: indent))
-            when 'worker'
-              section_builder.add_section(worker_build(cc, indent: indent))
-            when 'source'
-              section_builder.add_section(source_build(cc, indent: indent))
-            when 'filter'
-              section_builder.add_section(filter_build(cc, indent: indent))
-            when 'match'
-              section_builder.add_section(match_build(cc, indent: indent))
+            e.keys.each do |k|
+              cc = e.delete(k)
+              case k
+              when 'label'
+                section_builder.add_section(label_build(cc, indent: indent))
+              when 'worker'
+                section_builder.add_section(worker_build(cc, indent: indent))
+              when 'source'
+                section_builder.add_section(source_build(cc, indent: indent))
+              when 'filter'
+                section_builder.add_section(filter_build(cc, indent: indent))
+              when 'match'
+                section_builder.add_section(match_build(cc, indent: indent))
+              end
             end
           end
         end

data/lib/fluent/configurable.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/config/configure_proxy'
 require 'fluent/config/section'
 require 'fluent/config/error'

data/lib/fluent/daemon.rb

@@ -5,9 +5,10 @@ here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..'))
 
 require 'serverengine'
+require 'fluent/env'
 require 'fluent/supervisor'
 
 server_module = Fluent.const_get(ARGV[0])
 worker_module = Fluent.const_get(ARGV[1])
-params = JSON.parse(ARGV[2])
+params = JSON.parse(ARGV[2], **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
 ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }

data/lib/fluent/env.rb

@@ -26,6 +26,7 @@ module Fluent
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
+  DEFAULT_JSON_PARSE_OPTIONS = { allow_duplicate_key: true, allow_comments: true }.freeze
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
   INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid

data/lib/fluent/event.rb

@@ -309,6 +309,43 @@ module Fluent
     end
   end
 
+  class MetadataTimeEventStream < EventStream
+    def initialize(es)
+      @es = es
+    end
+
+    def empty?
+      @es.empty?
+    end
+
+    def dup
+      self.class.new(@es.dup)
+    end
+
+    def size
+      @es.size
+    end
+
+    def repeatable?
+      @es.repeatable?
+    end
+
+    def slice(index, num)
+      self.class.new(@es.slice(index, num))
+    end
+
+    def each(unpacker: nil, &block)
+      @es.each(unpacker: unpacker) do |time, record|
+        next if record.nil?
+
+        time = time[0] if time.is_a?(Array)
+        time = Fluent::EventTime.now if time.nil? || time.to_i == 0
+        block.call(time, record)
+      end
+      nil
+    end
+  end
+
   module ChunkMessagePackEventStreamer
     # chunk.extend(ChunkMessagePackEventStreamer)
     #  => chunk.each{|time, record| ... }

data/lib/fluent/load.rb

@@ -4,7 +4,6 @@ require 'time'
 require 'stringio'
 require 'fileutils'
 require 'json'
-require 'yajl'
 require 'uri'
 require 'msgpack'
 require 'strptime'