fluent-plugin-dict-map 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bd4e2441b2f79a58ae9613eaa8c2b1416b62ae71
+  data.tar.gz: a6d72c1e02117f5397623715cf45c44b9b867ca3
+SHA512:
+  metadata.gz: da87fd811c9dcb21b3de4b990862eefe6221f47309ee546b0250ce488291725583714c0147d429741070898792b84cdae82c0716d5651475b960654243189f46
+  data.tar.gz: b46e61c2961e75c84002c0cfb517473eecc1c84d20e08058876a17d9dbcd4807884aafb6c23e64a00f35e9477bb4a6259e7bb3b076467df76e6f581522728b82

data/.gitignore

@@ -0,0 +1,7 @@
+Gemfile.lock
+Gemfile.local
+README
+pkg/*
+test/*
+vendor/*
+.ruby-version

data/ChangeLog

@@ -0,0 +1,4 @@
+Release 0.1.0 - 2018/03/05
+
+  * First release
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,83 @@
+# Dictionary based map filter for [Fluentd](http://fluentd.org)
+
+Map field to other value with dictionary
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-dict-map
+
+## Configuration
+
+Use `@type dict_map` in `<filter>`.
+
+    <filter pattern>
+      @type dict_map
+      key_name level
+      dictionary {"I":"INFO","W":"WARN","E":"ERROR"}
+    </filter>
+
+For example, if following record is passed,
+
+```js
+{"msg":"hello!","level":"I"}
+```
+
+then you got filtered record like below:
+
+```js
+{"msg":"hello!","level":"INFO"}
+```
+
+### key_name
+
+The target key name.
+
+### destination_key_name
+
+The destination key name for mapped result. If not specified, the value is overwritten in `key_name`.
+
+Example:
+
+    # configuration
+    destination_key_name mapped
+    # filtered record
+    {"msg":"hello!","level":"I","mapped":"INFO"}
+
+### default_value
+
+This value is used when incoming value is missing in the dictionary. If not specified, value mapping doesn't happen.
+
+    # configuration
+    default_value UNKNOWN
+    # incoming record
+    {"msg":"hello!","level":"Z"}
+    # filtered record
+    {"msg":"hello!","level":"UNKNOWN"}
+
+### dictionary
+
+The json dictionary for value mapping.
+
+### dictionary_path
+
+Use json file instead of in-place `dictionary` parameter. This is useful for large dictionary. File ext must be `json`.
+
+Example:
+
+    dictionary_path /path/to/dict.json
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td>Masahiro Nakagawa <repeatedly@gmail.com></td>
+  </tr>
+  <tr>
+    <td>Copyright</td><td>Copyright (c) 2018- Masahiro Nakagawa</td>
+  </tr>
+  <tr>
+    <td>License</td><td>Apache 2 License</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/example/dict.json

@@ -0,0 +1,5 @@
+{
+  "foo" : "bar",
+  "num" : "100",
+  "10"  : "ten"
+}

data/fluent-plugin-dict-map.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-dict-map"
+  gem.description = "Dictionary based map filter for Fluentd"
+  gem.homepage    = "https://github.com/repeatedly/fluent-plugin-record-dict-map"
+  gem.summary     = gem.description
+  gem.version     = "0.1.0"
+  gem.authors     = ["Masahiro Nakagawa"]
+  gem.email       = "repeatedly@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 1.0", "< 2"]
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", "~> 3.1"
+end

data/lib/fluent/plugin/filter_dict_map.rb

@@ -0,0 +1,50 @@
+require 'fluent/plugin/filter'
+require 'json'
+
+module Fluent::Plugin
+  class DictMapFilter < Filter
+    Fluent::Plugin.register_filter('dict_map', self)
+
+    config_param :key_name, :string, :desc => 'The source key name'
+    config_param :destination_key_name, :string, :default => nil, :desc => 'The destination key name for mapped result'
+    config_param :default_value, :string, :default => nil, :desc => 'The value is used when incoming value is missing in the dictionary'
+    config_param :dictionary, :hash, :default => nil, :desc => 'The json dictionary for value mapping'
+    config_param :dictionary_path, :string, :default => nil, :desc => 'The path of dictionary file. File ext must be `json`'
+
+    def configure(conf)
+      super
+
+      if !@dictionary and !@dictionary_path
+        raise Fluent::ConfigError, "dictionary or dictionary_path parameter is required"
+      end
+
+      @dict = @dictionary ? @dictionary : load_dict(@dictionary_path)
+      @target = @destination_key_name.nil? ? @key_name : @destination_key_name
+    end
+
+    def filter(tag, time, record)
+      value = record[@key_name]
+      return record unless value
+
+      if @dict.include?(value)
+        record[@target] = @dict[value]
+      elsif !@default_value.nil?
+        record[@target] = @default_value
+      end
+
+      record
+    end
+
+    private
+
+    def load_dict(dict_path)
+      case
+      when dict_path.end_with?(".json")
+        require 'json'
+        JSON.parse(File.read(dict_path))
+      else
+        raise Fluent::ConfigError, "Invalid file type. Supported type is only JSON: #{dict_path}"
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-dict-map
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Masahiro Nakagawa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+description: Dictionary based map filter for Fluentd
+email: repeatedly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- example/dict.json
+- fluent-plugin-dict-map.gemspec
+- lib/fluent/plugin/filter_dict_map.rb
+homepage: https://github.com/repeatedly/fluent-plugin-record-dict-map
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Dictionary based map filter for Fluentd
+test_files: []