fluent-plugin-json-by-name 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: de1b7254b02e90e313f58420fc68d907c3bf430f03f6cd4e5e9c9fb5817114d2
+  data.tar.gz: 231703f04dda952f44f30af034ba167a5939498dec830f0e0377b2a4e0501a3a
+SHA512:
+  metadata.gz: 359a8700afc9d93d1648faf8d88a24e88ac0a790725eaf0c6d1e07adab794566b4f1895a2ec9a9572ba9191a9e1ec8775a89dda1c34c73b2b82d940eaa419013
+  data.tar.gz: e57c8a3305e04187ce93dfb9345ff456d95f5365d8d0229ec4dd2b613b008d42ab4a4dc1241e1d0e5b6f1bc6d97f10db327cd9933f603459bb2a6eb845196b73

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-json-by-name (0.1.0)
+      fluentd (~> 1.15, >= 1.15.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.2.2)
+    cool.io (1.7.1)
+    fluentd (1.15.3)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.4, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.8.0)
+    msgpack (1.6.1)
+    power_assert (2.0.2)
+    rake (13.0.6)
+    serverengine (2.3.2)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    test-unit (3.5.5)
+      power_assert
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2023.1)
+      tzinfo (>= 1.0.0)
+    webrick (1.7.0)
+    yajl-ruby (1.4.3)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  fluent-plugin-json-by-name!
+  rake (~> 13.0, >= 13.0.6)
+  test-unit (~> 3.1, >= 3.1.0)
+
+BUNDLED WITH
+   2.3.7

data/Makefile

@@ -0,0 +1,43 @@
+.PHONY: deps
+deps: deps-npm deps-ruby
+
+.PHONY: deps-npm
+deps-npm: node_modules
+
+node_modules: package.json
+	npm install
+
+.PHONY: deps-ruby
+deps-ruby:
+	bundle install
+
+.PHONY: lint
+lint: lint-md
+
+.PHONY: lint-fix
+lint-fix: lint-md-fix
+
+.PHONY: lint-md
+lint-md: node_modules
+	npx remark .
+
+.PHONY: lint-md-fix
+lint-md-fix: node_modules
+	npx remark . -o
+
+.PHONY: build
+build: build-ruby
+
+.PHONY: build-ruby
+build-ruby:
+	rake build
+
+.PHONY: test
+test: test-ruby
+
+.PHONY: test-ruby
+test-ruby:
+	rake test
+
+.PHONY: all
+all: deps-ruby build

data/README.md

@@ -0,0 +1,67 @@
+# fluent-plugin-json-by-name
+
+## what does this plugin do
+
+With this plugin you can add json fields as fluent field by the name of the
+json field
+
+## Configuration
+
+For just indexing, provide the fields with value null in json:
+
+```xml
+<filter **>
+  @type json_by_name
+  fields_to_index {
+    "test1": null,
+    "test2": null,
+    "test3": null
+  }
+</filter>
+```
+
+If you also want to rename the field, provide a new name as string instead of
+null. All this also works for nested fields:
+
+```xml
+<filter **>
+  @type json_by_name
+  fields_to_index {
+    "test1": null,
+    "test2": "this_was_field_test2",
+    "nested": {
+      "test3": null,
+      "test4": "this_was_nested_field_test4"
+    }
+  }
+</filter>
+```
+
+Assuming the above config, then the plugin would act as follows:
+
+### Inputs
+
+```json
+{"message": "{\"test1\": true}"}
+{"message": "{\"test2\": true}"}
+{"message": "{\"nested\": {\"test3\": true}}"}
+{"message": "{\"nested\": {\"test4\": true}}"}
+```
+
+### Outputs
+
+```json
+{"message": "{\"test1\": true}", "test1": true}
+{"message": "{\"test2\": true}", "this_was_field_test2": true}
+{"message": "{\"nested\": {\"test3\": true}}", "nested.test3": true}
+{"message": "{\"nested\": {\"test4\": true}}", "this_was_nested_field_test4": true}
+```
+
+More examples can be found in `test/plugin/test_filter_json_by_name.rb`
+
+## Testing
+
+```sh
+bundle install
+rake
+```

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler"
+require "rake/testtask"
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/build.sh

@@ -0,0 +1,2 @@
+bundle install
+bundle exec rake build

data/fluent_plugin_json_by_name.gemspec

@@ -0,0 +1,12 @@
+require_relative 'lib/release/fluent_plugin_json_by_name/version'
+
+Gem::Specification.new do |gem|
+  gem.name = 'fluent-plugin-json-by-name'
+  gem.version = FluentPluginJsonByName::VERSION
+  gem.summary = "Fluentd filter plugin to reshape json"
+  gem.authors = "platform+rubygems@riskident.com"
+  gem.files = `find *`.split("\n").uniq.sort.select{|f| !f.empty? }
+  gem.add_runtime_dependency     'fluentd', '~> 1.15', '>= 1.15.3'
+  gem.add_development_dependency 'rake', '~> 13.0', '>= 13.0.6'
+  gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'
+end

data/lib/fluent/plugin/filter_json_by_name.rb

@@ -0,0 +1,62 @@
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+
+  class JsonByName < Filter
+    Fluent::Plugin.register_filter('json_by_name', self)
+
+    helpers :record_accessor
+
+    desc 'A hash/JSON object of fields to index and rename from log messages.'
+    config_param :fields_to_index, :hash, default: nil
+
+    def configure(conf)
+      super
+      validate_config(fields_to_index)
+    end
+
+    def combine_path(path, element)
+      return element if path.nil?
+      return "#{path}.#{element}"
+    end
+
+    def validate_config(fields_to_index_remaining, path=nil)
+      if fields_to_index_remaining.class == Hash then
+        fields_to_index_remaining.keys.each do |key|
+          validate_config(fields_to_index_remaining[key], combine_path(path, key))
+        end
+        return
+      elsif fields_to_index_remaining.nil? or fields_to_index_remaining.class == String then
+        return
+      end
+      raise Fluent::ConfigError.new("value at \"#{path}\" in config is not either object, string, or null")
+    end
+
+    def reshape(record, fields_to_index_remaining, data, path=nil)
+      return if data == nil
+      if fields_to_index_remaining.class == Hash and data.class == Hash then
+        common_keys = Set.new(fields_to_index_remaining.keys) & Set.new(data.keys)
+        common_keys.each do |key|
+          reshape(record, fields_to_index_remaining[key], data[key], combine_path(path, key))
+        end
+      elsif fields_to_index_remaining.nil? then
+        record[path] = data
+      elsif fields_to_index_remaining.class == String then
+        record[fields_to_index_remaining] = data
+      end
+    end
+
+    def filter(tag, time, record)
+      return record if record["message"] == nil
+      begin
+        record_as_json = JSON.parse(record["message"])
+      rescue JSON::ParserError
+        return record
+      end
+
+      reshape(record, fields_to_index, record_as_json)
+
+      return record
+    end
+  end
+end

data/lib/release/fluent_plugin_json_by_name/version.rb

@@ -0,0 +1,3 @@
+module FluentPluginJsonByName
+  VERSION = '0.1.0'.freeze
+end

data/package.json

@@ -0,0 +1,9 @@
+{
+  "remarkConfig": ".remarkrc",
+  "devDependencies": {
+    "remark-cli": "^11.0.0",
+    "remark-gfm": "^3.0.1",
+    "remark-lint": "^9.1.1",
+    "remark-preset-lint-markdown-style-guide": "^5.1.2"
+  }
+}

data/publish.sh

@@ -0,0 +1,3 @@
+mkdir -p -- /root/.gem/
+gem build *.gemspec
+gem push *.gem

data/test/helper.rb

@@ -0,0 +1,12 @@
+require 'bundler/setup'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
+require 'fluent/test'
+require 'fluent/test/driver/filter'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+
+require 'fluent/plugin/filter_json_by_name'

data/test/plugin/test_filter_json_by_name.rb

@@ -0,0 +1,92 @@
+require 'helper'
+
+class JsonByNameTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  fields_to_index {
+    "test1": null,
+    "test2": "this_was_field_test2",
+    "nested": {
+      "test3": null,
+      "test4": "this_was_field_test4"
+    },
+    "nonnested.test5": null,
+    "nonnested.test6": "this_was_field_test6"
+  }
+  ]
+
+  def create_driver(config)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::JsonByName).configure(config)
+  end
+
+  def run_plugin(message, config = CONFIG) 
+    d = create_driver(config)
+    d.run(default_tag: "test") do
+      d.feed(Fluent::Engine.now, {"message" => message})
+    end
+    return d.filtered_records
+  end
+
+  def assert_message(original_message, result, added_fields)
+    message = {"message" => original_message}
+    full_message = message.merge(added_fields)
+    assert_equal([full_message], result)
+  end
+
+  def run_and_assert(message, added_fields, config = CONFIG)
+    result = run_plugin(message, config)
+    assert_message(message, result, added_fields)
+  end
+
+  test 'valid config validates' do
+    assert_nothing_raised(RuntimeError) {
+      create_driver(CONFIG)
+    }
+  end
+
+  test 'invalid config fails validation' do
+    assert_raise(Fluent::ConfigError.new("value at \"foo.invalid_config\" in config is not either object, string, or null")) {
+      create_driver(<<-CONFIG)
+      fields_to_index {
+        "foo": {
+          "rename_given_name": "renamename",
+          "invalid_config"   : true,
+          "rename_same_name" : null
+        }
+      }
+      CONFIG
+    }
+  end
+
+  test 'invalid json is left as is' do
+    run_and_assert("this_is_not_json", {})
+  end
+
+  test 'indexing field' do
+    run_and_assert("{\"test1\":true}", {"test1" => true})
+  end
+
+  test 'indexing and renaming field' do
+    run_and_assert("{\"test2\":true}", {"this_was_field_test2" => true})
+  end
+
+  test 'indexing nested field' do
+    run_and_assert("{\"nested\":{\"test3\":true}}", {"nested.test3" => true})
+  end
+
+  test 'indexing and renaming nested field' do
+    run_and_assert("{\"nested\":{\"test4\":true}}", {"this_was_field_test4" => true})
+  end
+
+  test 'indexing field with "." in name' do
+    run_and_assert("{\"nonnested.test5\":true}", {"nonnested.test5" => true})
+  end
+
+  test 'indexing and renaming field with "." in name' do
+    run_and_assert("{\"nonnested.test6\":true}", {"this_was_field_test6" => true})
+  end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-json-by-name
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- platform+rubygems@riskident.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.15.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.15.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- Makefile
+- README.md
+- Rakefile
+- build.sh
+- fluent_plugin_json_by_name.gemspec
+- lib/fluent/plugin/filter_json_by_name.rb
+- lib/release/fluent_plugin_json_by_name/version.rb
+- package.json
+- pkg/fluent-plugin-json-by-name-0.1.0.gem
+- publish.sh
+- test/helper.rb
+- test/plugin/test_filter_json_by_name.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key: 
+specification_version: 4
+summary: Fluentd filter plugin to reshape json
+test_files: []