fluent-plugin-uri-parser 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 529c3e22f0bb6e97190a8db3c9bccd8005db2eb0
-  data.tar.gz: 7a61cff618e86c55178494fd51a528ea7daedf80
+SHA256:
+  metadata.gz: 159e54e1d62cc98d696b44b59550cb54fd02f9a202afcdf55e5e8df2c7f39996
+  data.tar.gz: ed7ee9ce893e2e0cdbbff3fe8ab616b7efcb23db1071a1809a2bb9021cf1d1ad
 SHA512:
-  metadata.gz: c4e6a42fc6d43a03d32c45e7017a16597e6354e1a3b05508a61f58b57ea5da060236f314abe447b454b010f17f5cba639a6af377e7d5123d94f599c1adefacb1
-  data.tar.gz: bfc2f8c5f4184ed49fc746e564ae5aa97428b071c7c0929001a51c3835896b6598a0fb276167db76afe2a1668fbd2e040bf47d737fa08d14efa14605afe78ac6
+  metadata.gz: 0a91925202870044e0099870e59de97670fc80591373b0d13db486827d5e8104188f4828f7ce13fcc4d731ee9b5e887c3320338b28bd3abf84029273eba8bbae
+  data.tar.gz: 9968c9681ebccc76584994f3b5479d62ce12a4c7e41f522f428c38697fc2374a453853082ab72f94eb96813bba2388e43f7a88944417d2a29537733f43d17400

data/.github/workflows/test.yml

@@ -0,0 +1,28 @@
+name: test
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - "3.2"
+          - "3.3"
+          - "3.4"
+          - "4.0"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run tests
+        run: bundle exec rake test

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/vendor/

data/README.md

@@ -1,104 +1,266 @@
 # fluent-plugin-uri-parser
 
-[![Gem Version](https://badge.fury.io/rb/fluent-plugin-uri-parser.svg)](https://badge.fury.io/rb/fluent-plugin-uri-parser) [![wercker status](https://app.wercker.com/status/a735d29143f3a1a727fc65653bc81e2a/s "wercker status")](https://app.wercker.com/project/bykey/a735d29143f3a1a727fc65653bc81e2a)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-uri-parser.svg)](https://badge.fury.io/rb/fluent-plugin-uri-parser)
+[![test](https://github.com/daichirata/fluent-plugin-uri-parser/actions/workflows/test.yml/badge.svg)](https://github.com/daichirata/fluent-plugin-uri-parser/actions/workflows/test.yml)
 
-This is a Fluentd plugin to parse uri and query string in log messages.
+Fluentd filter plugins that **decompose URIs and query strings into structured fields**, so you can search, group, and aggregate on them in your downstream stack.
+
+```text
+"https://example.com:8080/search?q=fluentd&lang=ja"
+                                ↓
+{ scheme: "https", host: "example.com", port: 8080,
+  path: "/search", query: "q=fluentd&lang=ja", fragment: nil }
+```
+
+## Why?
+
+URL strings sitting in a single log field are a black box — you can't filter by host, group by path, or count by query parameter without parsing them first. These filters turn a raw URL into a record your storage can index.
+
+| Plugin | Turns this | Into this |
+|---|---|---|
+| `uri_parser` | `https://example.com:8080/p?q=1#x` | `scheme`, `host`, `port`, `path`, `query`, `fragment` |
+| `query_string_parser` | `foo=bar&hoge=fuga` | `{ "foo" => "bar", "hoge" => "fuga" }` |
 
 ## Requirements
 
-| fluent-plugin-uri-parser  | fluentd    | ruby   |
-|---------------------------|------------|--------|
-| >= 0.3.0                  | >= v0.14.0 | >= 2.1 |
-|  < 0.2.0                  | >= v0.12.0 | >= 1.9 |
+| fluent-plugin-uri-parser | fluentd   | ruby   |
+| ------------------------ | --------- | ------ |
+| >= 0.4.0                 | >= v1.0.0 | >= 3.2 |
 
 ## Installation
 
-``` shell
-$ gem install fluent-plugin-uri-parser -v "~> 0.2"  # for fluentd v0.12 or later
-$ gem install fluent-plugin-uri-parser              # for fluentd v0.14 or later
+```shell
+gem install fluent-plugin-uri-parser
 ```
 
-## Component
+Or in your `Gemfile`:
+
+```ruby
+gem "fluent-plugin-uri-parser"
+```
 
-### URIParserFilter
+---
 
-This is a Fluentd plugin to parse and filtering uri in log messages and re-emit them.
+## `uri_parser`
 
-### QueryStringParserFilter
+Decomposes a URI field into its components.
 
-This is a Fluentd plugin to parse and filtering query string in log messages and re-emit them.
+### Minimal example
 
-## Configuration
+```aconf
+<filter access.log>
+  @type uri_parser
+  key_name url
+  out_key_scheme   scheme
+  out_key_host     host
+  out_key_port     port
+  out_key_path     path
+  out_key_query    query
+  out_key_fragment fragment
+</filter>
+```
 
+```jsonc
+// input
+{ "url": "https://example.com:8080/search?q=fluentd#top" }
+
+// output
+{
+  "url":      "https://example.com:8080/search?q=fluentd#top",
+  "scheme":   "https",
+  "host":     "example.com",
+  "port":     8080,
+  "path":     "/search",
+  "query":    "q=fluentd",
+  "fragment": "top"
+}
 ```
-<filter>
+
+> The `port` value uses [`Addressable::URI#inferred_port`](https://github.com/sporkmonger/addressable), so well-known schemes (`http` → 80, `https` → 443, ...) get a port even when the URL omits one.
+
+### Group output under a single key — `hash_value_field`
+
+```aconf
+<filter access.log>
   @type uri_parser
-  key_name uri
-  inject_key_prefix parsed
-  # hash_value_field parsed
-  # suppress_parse_error_log false
-  # ignore_key_not_exist false
-  # ignore_nil false
-
-  out_key_scheme scheme
+  key_name url
+  hash_value_field parsed
   out_key_host host
-  out_key_port port
   out_key_path path
-  out_key_query query
-  out_key_fragment fragment
-</match>
-# input string of data: {"uri": "http://example.com/path?foo=bar#t=1"}
-# output data: {"parsed.scheme":"http","parsed.host":"example.com","parsed.port":80,"parsed.path":"/path","parsed.query":"foo=bar","parsed.ragment":"t=1"}
+</filter>
+```
+
+```jsonc
+// input
+{ "url": "https://example.com/search" }
+
+// output
+{
+  "url": "https://example.com/search",
+  "parsed": { "host": "example.com", "path": "/search" }
+}
+```
+
+### Namespace output keys — `inject_key_prefix`
+
+```aconf
+<filter access.log>
+  @type uri_parser
+  key_name url
+  inject_key_prefix url.
+  out_key_host host
+  out_key_path path
+</filter>
+```
+
+```jsonc
+// input
+{ "url": "https://example.com/search" }
+
+// output
+{
+  "url":      "https://example.com/search",
+  "url.host": "example.com",
+  "url.path": "/search"
+}
+```
+
+### Drop empty components — `ignore_nil`
 
-<filter>
+When a component is missing (no query, no fragment, etc.) the default is to emit it as `null`. Set `ignore_nil true` to omit those keys entirely.
+
+```jsonc
+// input
+{ "url": "https://example.com/path" }
+
+// ignore_nil false  (default)
+{ "scheme": "https", "host": "example.com", "port": 443,
+  "path": "/path", "query": null, "fragment": null }
+
+// ignore_nil true
+{ "scheme": "https", "host": "example.com", "port": 443,
+  "path": "/path" }
+```
+
+---
+
+## `query_string_parser`
+
+Decomposes a query string field into individual parameters.
+
+> Pairs with an empty key (e.g. the leading `&` in `&foo=1`) are silently dropped — they're noise from user-supplied URLs and never represent a real parameter.
+
+### Minimal example
+
+```aconf
+<filter access.log>
+  @type query_string_parser
+  key_name query
+</filter>
+```
+
+```jsonc
+// input
+{ "query": "foo=bar&hoge=fuga" }
+
+// output
+{ "query": "foo=bar&hoge=fuga", "foo": "bar", "hoge": "fuga" }
+```
+
+### Group output under a single key — `hash_value_field`
+
+```aconf
+<filter access.log>
   @type query_string_parser
-  key_name parsed.query
-  hash_value_field query
-  # inject_key_prefix query
-  # suppress_parse_error_log false
-  # ignore_key_not_exist false
-</match>
-# input string of data: {"parsed.query": "foo=bar"}
-# output data: {"query":{"foo":"bar"}}
+  key_name query
+  hash_value_field params
+</filter>
+```
 
+```jsonc
+// input
+{ "query": "foo=bar&hoge=fuga" }
+
+// output
+{
+  "query":  "foo=bar&hoge=fuga",
+  "params": { "foo": "bar", "hoge": "fuga" }
+}
 ```
 
-**key_name (Required)**
+### Handle repeated parameters
+
+A request like `?tag=ruby&tag=fluentd` has two `tag` values. By default the last one wins (scalar). You have two ways to keep both:
+
+#### Option A: always arrays — `multi_value_params true`
 
-Key of the value to be parsed in the record.
+Every parameter becomes an array, even when it appeared once.
 
-**hash_value_field (Default: '')**
+```jsonc
+// input
+{ "query": "tag=ruby&tag=fluentd&lang=ja" }
 
-If a value is set, the value after parsing is stored in hash with key specified value.
+// output
+{ "tag": ["ruby", "fluentd"], "lang": ["ja"] }
+```
 
-**inject_key_prefix (Default: '')**
+#### Option B: array only for listed names — `multi_value_param_names`
 
-If you set a value, set the value specified for the key after parsing as prefix.
+You know `tag` may repeat but `lang` won't. Keep `lang` as a scalar and only wrap `tag` in an array.
 
-**suppress_parse_error_log (Default: false)**
+```aconf
+<filter access.log>
+  @type query_string_parser
+  key_name query
+  multi_value_param_names tag
+</filter>
+```
 
-If set to `true`, no error log is output even if parsing fails.
+```jsonc
+// input
+{ "query": "tag=ruby&tag=fluentd&lang=ja" }
 
-**ignore_key_not_exist (Default: false)**
+// output
+{ "tag": ["ruby", "fluentd"], "lang": "ja" }
+```
 
-If set to `true`, if the field specified by `key_name` does not exist, the record will not be emit to the next stream. That means that the data will be lost there.
+> When both are set, `multi_value_params true` wins.
 
-**ignore_nil (Default: false)**
+---
 
-If set to `true`, exclude key if the value after parse is nil.
+## Options
 
-**multi_value_params (Default: false)**
+Shared between both filters unless noted.
 
-If set to `true`, then resulting values would be arrays containing
-potentially multiple values of a given parameter.
+| Option | Type | Default | What it does |
+| --- | --- | --- | --- |
+| `key_name` | string | — *(required)* | Record key holding the URL or query string to parse. |
+| `hash_value_field` | string | `nil` | If set, all extracted fields are nested under this key. |
+| `inject_key_prefix` | string | `nil` | Prefix prepended to every extracted key. |
+| `ignore_key_not_exist` | bool | `false` | When `key_name` is missing, drop the record instead of passing it through. |
+| `emit_invalid_record_to_error` | bool | `true` | When `key_name` is missing, emit the record to Fluentd's error stream. |
+| `suppress_parse_error_log` | bool | `false` | Silence the warning log when the value fails to parse. |
+| `ignore_nil` | bool | `false` | *(uri_parser only)* Omit output keys whose parsed value is `nil`. |
+| `out_key_scheme` / `out_key_host` / `out_key_port` / `out_key_path` / `out_key_query` / `out_key_fragment` | string | `nil` | *(uri_parser only)* Output key name for each URI component. Components without an `out_key_*` are not emitted. |
+| `multi_value_params` | bool | `false` | *(query_string_parser only)* Emit every parameter as an array. |
+| `multi_value_param_names` | array | `nil` | *(query_string_parser only)* Emit only the listed parameters as arrays. Ignored when `multi_value_params` is `true`. |
+
+---
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+```shell
+bundle install
+bundle exec rake test
+```
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine: `bundle exec rake install`.
+To release: bump the version in the gemspec, then `bundle exec rake release` (tags, pushes, and uploads to [rubygems.org](https://rubygems.org)).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/daichirata/fluent-plugin-uri-parser.
+Bug reports and pull requests are welcome at <https://github.com/daichirata/fluent-plugin-uri-parser>.
+
+## License
 
+Apache-2.0

data/fluent-plugin-uri-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-uri-parser"
-  spec.version       = "0.3.0"
+  spec.version       = "0.4.0"
   spec.authors       = ["Daichi HIRATA"]
   spec.email         = ["daichirata@gmail.com"]
   spec.license       = "Apache-2.0"
@@ -13,14 +13,16 @@ Gem::Specification.new do |spec|
   spec.description   = "This is a Fluentd plugin to parse uri and query string in log messages."
   spec.homepage      = "https://github.com/daichirata/fluent-plugin-uri-parser"
 
+  spec.required_ruby_version = ">= 3.2"
+
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 1.0", "< 2"]
   spec.add_runtime_dependency "addressable"
 
-  spec.add_development_dependency "test-unit"
-  spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "test-unit", "~> 3.6"
 end

data/lib/fluent/plugin/filter_query_string_parser.rb

@@ -1,3 +1,6 @@
+require 'fluent/plugin/filter'
+require 'addressable/uri'
+
 module Fluent
   module Plugin
     class QueryStringParserFilter < Filter
@@ -10,11 +13,7 @@ module Fluent
       config_param :ignore_key_not_exist, :bool, default: false
       config_param :emit_invalid_record_to_error, :bool, default: true
       config_param :multi_value_params, :bool, default: false
-
-      def initialize
-        super
-        require "uri"
-      end
+      config_param :multi_value_param_names, :array, default: nil
 
       def filter(tag, time, record)
         raw_value = record[@key_name]
@@ -27,12 +26,22 @@ module Fluent
         end
 
         begin
-          params = URI.decode_www_form(raw_value)
+          params = Addressable::URI.form_unencode(raw_value)
+          params = params.reject {|k, _| k.nil? || k.empty? }
 
           unless params.empty?
             if @multi_value_params
               values = Hash.new {|h,k| h[k] = [] }
               params.each{|pair| values[pair[0]].push(pair[1])}
+            elsif @multi_value_param_names
+              values = {}
+              params.each do |k, v|
+                if @multi_value_param_names.include?(k)
+                  (values[k] ||= []) << v
+                else
+                  values[k] = v
+                end
+              end
             else
               values = Hash[params]
             end

data/lib/fluent/plugin/filter_uri_parser.rb

@@ -1,3 +1,6 @@
+require 'fluent/plugin/filter'
+require 'addressable/uri'
+
 module Fluent
   module Plugin
     class URIParserFilter < Filter
@@ -18,11 +21,6 @@ module Fluent
       config_param :out_key_query, :string, default: nil
       config_param :out_key_fragment, :string, default: nil
 
-      def initialize
-        super
-        require "addressable/uri"
-      end
-
       def filter(tag, time, record)
         raw_value = record[@key_name]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-uri-parser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Daichi HIRATA
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-28 00:00:00.000000000 Z
+date: 2026-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -45,7 +45,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,33 +59,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.6'
 description: This is a Fluentd plugin to parse uri and query string in log messages.
 email:
 - daichirata@gmail.com
@@ -93,6 +93,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - Gemfile
 - LICENSE.txt
@@ -101,12 +102,11 @@ files:
 - fluent-plugin-uri-parser.gemspec
 - lib/fluent/plugin/filter_query_string_parser.rb
 - lib/fluent/plugin/filter_uri_parser.rb
-- wercker.yml
 homepage: https://github.com/daichirata/fluent-plugin-uri-parser
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -114,16 +114,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
-signing_key: 
+rubygems_version: 3.5.16
+signing_key:
 specification_version: 4
 summary: This is a Fluentd plugin to parse uri and query string in log messages.
 test_files: []

data/wercker.yml

@@ -1,25 +0,0 @@
-box: ruby:2.3.0
-# Build definition
-build:
-  steps:
-    - script:
-      name: update bundler
-      code: gem update bundler
-
-    # A step that executes `bundle install` command
-    - bundle-install:
-      jobs: 4
-
-    # A custom script step, name value is used in the UI
-    # and the code value contains the command that get executed
-    - script:
-      name: echo ruby information
-      code: |
-        echo "ruby version $(ruby --version) running"
-        echo "from location $(which ruby)"
-        echo -p "gem list: $(gem list)"
-
-    # Add more steps here:
-    - script:
-      name: test
-      code: bundle exec rake test