fluent-plugin-uri-cma-parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8c10992d5f384ddb2778a65b1a0da29102ad683e735ea02f3f124068f7db9a09
+  data.tar.gz: 2526dc16b541e89ef61d961446e41d5c4d3066fa00b82dfc2c629d8c5ec6da4b
+SHA512:
+  metadata.gz: ebaad4f371e53c04c8b1162b96651fbb2151a27df3d36aef31af86e4190b2b4a781beef1d3924006c6725c680460e5c27371aeaa3aa80c812cafd1a067b63b18
+  data.tar.gz: 537f66fc203fcd709e305864052ce042682c00bd66ef76c2b632ce016d5de5e2db83138543d4857ff67eb9f72837662a0d25488f6685ced478898f89345bd0bf

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/
+.idea
+*.iml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-url-parser.gemspec
+gemspec

data/README.md

@@ -0,0 +1,104 @@
+# fluent-plugin-uri-cma-parser
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-uri-parser.svg)](https://badge.fury.io/rb/fluent-plugin-uri-parser) [![wercker status](https://app.wercker.com/status/a735d29143f3a1a727fc65653bc81e2a/s "wercker status")](https://app.wercker.com/project/bykey/a735d29143f3a1a727fc65653bc81e2a)
+
+This is a Fluentd plugin to parse uri and query string in log messages.
+
+## Requirements
+
+| fluent-plugin-uri-cma-parser  | fluentd    | ruby   |
+|---------------------------|------------|--------|
+| >= 0.3.0                  | >= v0.14.0 | >= 2.1 |
+|  < 0.2.0                  | >= v0.12.0 | >= 1.9 |
+
+## Installation
+
+``` shell
+$ gem install fluent-plugin-uri-cma-parser -v "~> 0.2"  # for fluentd v0.12 or later
+$ gem install fluent-plugin-uri-cma-parser              # for fluentd v0.14 or later
+```
+
+## Component
+
+### URIParserFilter
+
+This is a Fluentd plugin to parse and filtering uri in log messages and re-emit them.
+
+### QueryStringParserFilter
+
+This is a Fluentd plugin to parse and filtering query string in log messages and re-emit them.
+
+## Configuration
+
+```
+<filter>
+  @type uri_parser
+  key_name uri
+  inject_key_prefix parsed
+  # hash_value_field parsed
+  # suppress_parse_error_log false
+  # ignore_key_not_exist false
+  # ignore_nil false
+
+  out_key_scheme scheme
+  out_key_host host
+  out_key_port port
+  out_key_path path
+  out_key_query query
+  out_key_fragment fragment
+</filter>
+# input string of data: {"uri": "http://example.com/path?foo=bar#t=1"}
+# output data: {"parsed.scheme":"http","parsed.host":"example.com","parsed.port":80,"parsed.path":"/path","parsed.query":"foo=bar","parsed.ragment":"t=1"}
+
+<filter>
+  @type query_string_parser
+  key_name parsed.query
+  hash_value_field query
+  # inject_key_prefix query
+  # suppress_parse_error_log false
+  # ignore_key_not_exist false
+</filter>
+# input string of data: {"parsed.query": "foo=bar"}
+# output data: {"query":{"foo":"bar"}}
+
+```
+
+**key_name (Required)**
+
+Key of the value to be parsed in the record.
+
+**hash_value_field (Default: '')**
+
+If a value is set, the value after parsing is stored in hash with key specified value.
+
+**inject_key_prefix (Default: '')**
+
+If you set a value, set the value specified for the key after parsing as prefix.
+
+**suppress_parse_error_log (Default: false)**
+
+If set to `true`, no error log is output even if parsing fails.
+
+**ignore_key_not_exist (Default: false)**
+
+If set to `true`, if the field specified by `key_name` does not exist, the record will not be emit to the next stream. That means that the data will be lost there.
+
+**ignore_nil (Default: false)**
+
+If set to `true`, exclude key if the value after parse is nil.
+
+**multi_value_params (Default: false)**
+
+If set to `true`, then resulting values would be arrays containing
+potentially multiple values of a given parameter.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/daichirata/fluent-plugin-uri-cma-parser.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-uri-parser.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-uri-cma-parser"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Philippe BENAOUN"]
+  spec.email         = ["philippe.benaoun@gmail.com"]
+  spec.license       = "Apache-2.0"
+
+  spec.summary       = "This is a Fluentd plugin to parse uri and query string in log messages."
+  spec.description   = "This is a Fluentd plugin to parse uri and query string in log messages."
+  spec.homepage      = "https://gitlab.com/pbenaoun/fluent-plugin-uri-cma-parser"
+
+  ##spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
+  spec.add_runtime_dependency "addressable"
+
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fluent/plugin/filter_query_string_parser.rb

@@ -0,0 +1,79 @@
+module Fluent
+  module Plugin
+    class QueryStringParserFilter < Filter
+      Fluent::Plugin.register_filter("query_string_parser", self)
+
+      config_param :out_key_path, :string, default: 'path'
+      config_param :out_key_query, :string, default: 'params'
+      config_param :out_key_fragment, :string, default: 'fragment'
+      config_param :key_name, :string
+      config_param :hash_value_field, :string, default: 'indexing'
+      config_param :inject_key_prefix, :string, default: nil
+      config_param :suppress_parse_error_log, :bool, default: false
+      config_param :ignore_key_not_exist, :bool, default: false
+      config_param :emit_invalid_record_to_error, :bool, default: true
+      config_param :multi_value_params, :bool, default: false
+
+      def initialize
+        super
+        require "addressable/uri"
+      end
+
+      def cleaner_prefix(line)
+        line.gsub(Regexp.new('^.*(\?)'), '').strip
+      end
+      
+      def valid_url(line)
+        if line.match(Regexp.new('(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})'))
+          true
+        else
+          false
+        end
+      end
+
+      def filter(tag, time, record)
+        raw_value = record[@key_name]
+
+        if raw_value.nil?
+          if @emit_invalid_record_to_error
+            router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+          end
+          return @ignore_key_not_exist ? nil : record
+        end
+
+        begin
+          if valid_url(raw_value)
+            uri = Addressable::URI.parse(raw_value)
+            values = {}
+            values[@out_key_path] = uri.path if @out_key_path
+            values[@out_key_query] = uri.query if @out_key_query
+            values[@out_key_fragment] = uri.fragment if @out_key_fragment
+            values.reject! {|_, v| v.nil? } if @ignore_nil
+            unless values[@out_key_query].nil?
+              params = Addressable::URI.form_unencode(cleaner_prefix(values[@out_key_query]))
+            end
+
+            unless params.empty?
+              if @multi_value_params
+                values = Hash.new {|h,k| h[k] = [] }
+                params.each{|pair| values[pair[0]].push(pair[1])}
+              else
+                values = Hash[params]
+              end
+              if @inject_key_prefix
+                values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+              end
+              r = @hash_value_field ? { @hash_value_field => values } : values
+              record = record.merge(r)
+            end
+          else
+            log.warn "url not valid: #{raw_value}"
+          end
+        rescue => e
+          log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+        end
+        record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_uri_parser.rb

@@ -0,0 +1,63 @@
+module Fluent
+  module Plugin
+    class URIParserFilter < Filter
+      Fluent::Plugin.register_filter("uri_parser", self)
+
+      config_param :key_name, :string
+      config_param :hash_value_field, :string, default: nil
+      config_param :inject_key_prefix, :string, default: nil
+      config_param :suppress_parse_error_log, :bool, default: false
+      config_param :ignore_key_not_exist, :bool, default: false
+      config_param :ignore_nil, :bool, default: false
+      config_param :emit_invalid_record_to_error, :bool, default: true
+
+      config_param :out_key_scheme, :string, default: nil
+      config_param :out_key_host, :string, default: nil
+      config_param :out_key_port, :string, default: nil
+      config_param :out_key_path, :string, default: nil
+      config_param :out_key_query, :string, default: nil
+      config_param :out_key_fragment, :string, default: nil
+
+      def initialize
+        super
+        require "addressable/uri"
+      end
+
+      def filter(tag, time, record)
+        raw_value = record[@key_name]
+
+        if raw_value.nil?
+          if @emit_invalid_record_to_error
+            router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+          end
+          return @ignore_key_not_exist ? nil : record
+        end
+
+        begin
+          uri = Addressable::URI.parse(raw_value)
+
+          values = {}
+          values[@out_key_scheme] = uri.scheme if @out_key_scheme
+          values[@out_key_host] = uri.host if @out_key_host
+          values[@out_key_port] = uri.inferred_port if @out_key_port
+          values[@out_key_path] = uri.path if @out_key_path
+          values[@out_key_query] = uri.query if @out_key_query
+          values[@out_key_fragment] = uri.fragment if @out_key_fragment
+          values.reject! {|_, v| v.nil? } if @ignore_nil
+
+          unless values.empty?
+            if @inject_key_prefix
+              values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+            end
+            r = @hash_value_field ? { @hash_value_field => values } : values
+            record = record.merge(r)
+          end
+        rescue => e
+          log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+        end
+
+        return record
+      end
+    end
+  end
+end

data/wercker.yml

@@ -0,0 +1,25 @@
+box: ruby:2.3.0
+# Build definition
+build:
+  steps:
+    - script:
+      name: update bundler
+      code: gem update bundler
+
+    # A step that executes `bundle install` command
+    - bundle-install:
+      jobs: 4
+
+    # A custom script step, name value is used in the UI
+    # and the code value contains the command that get executed
+    - script:
+      name: echo ruby information
+      code: |
+        echo "ruby version $(ruby --version) running"
+        echo "from location $(which ruby)"
+        echo -p "gem list: $(gem list)"
+
+    # Add more steps here:
+    - script:
+      name: test
+      code: bundle exec rake test

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-uri-cma-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Philippe BENAOUN
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-03-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This is a Fluentd plugin to parse uri and query string in log messages.
+email:
+- philippe.benaoun@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-uri-parser.gemspec
+- lib/fluent/plugin/filter_query_string_parser.rb
+- lib/fluent/plugin/filter_uri_parser.rb
+- wercker.yml
+homepage: https://gitlab.com/pbenaoun/fluent-plugin-uri-cma-parser
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: This is a Fluentd plugin to parse uri and query string in log messages.
+test_files: []