fluent-plugin-json-lookup 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ddf77b69fc683ba857cc1f189d8b39abf3971f9f
+  data.tar.gz: 99808744272c71e1021d715fb498b7b642865787
+SHA512:
+  metadata.gz: 6e77efe375a623c907f4027f36620f4ba15a8a8535385aeb868ec5c9286daf9740c8b7ce5253fb5b1eecc080d618769c0253d893a78fb4364f937b1d914c2225
+  data.tar.gz: 3e85a8d7377904e1b6220cef6766149b1d42fecd694f2596bf8d7a22f2ec678358d2e560b5f967c00a7d1b96d3cf9766a33dce5c194728ef3cca11bccb14da9f

data/.gitignore

@@ -0,0 +1,7 @@
+# Package
+pkg/
+
+*.gem
+
+*.swp
+.idea/

data/.rubocop.yml

@@ -0,0 +1,17 @@
+Metrics/LineLength:
+  Max: 120
+Documentation:
+  Enabled: false
+
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 131
+
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 182
+
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 29

data/.travis.yml

@@ -0,0 +1,23 @@
+language: ruby
+
+rvm:
+  - 2.2
+  - 2.3
+  - 2.4
+  - ruby-head
+
+gemfile:
+ - Gemfile
+ - Gemfile.v0.12
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+
+before_install:
+- gem update bundler
+- gem install rubocop
+
+script:
+- rubocop
+- bundle exec rake test

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify gem's dependencies in fluent-plugin-json-lookup.gemspec
+
+gemspec

data/Gemfile.v0.12

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify gem's dependencies in fluent-plugin-json-lookup.gemspec
+gem 'fluentd', '~> 0.12.0'
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Skuid, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,101 @@
+# fluent-plugin-json-lookup
+
+Fluentd filter plugin for looking up json objects from messages.
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-json-lookup
+
+## Configuration
+
+### Use  of the `json_lookup` filter.
+
+```
+<filter pattern>
+  @type json_lookup
+  lookup_key kubernetes_container_name
+  json_key kubernetes_annotations_fluentd_org/keys
+  remove_json_key true
+</filter>
+```
+
+If following record is passed:
+
+```json
+{
+    "kubernetes_container_name" : "nginx",
+    "kubernetes_annotations_fluentd_org/keys" : "{\"nginx\": {\"es_type\": \"nginx\", \"es_index\": \"fluentd-nginx-\"}, \"webapp\": {\"es_type\": \"application\", \"es_index\": \"fluentd-app-\"}}"
+}
+```
+
+then the emitted record would be:
+
+```json
+{
+    "kubernetes_container_name": "nginx",
+    "es_type": "nginx",
+    "es_index": "fluentd-nginx-"
+}
+```
+
+### Alternate use of the `json_lookup` filter.
+
+```
+<filter pattern>
+  @type json_lookup
+  lookup_key nginx
+  json_key kubernetes_annotations_fluentd_org/keys
+  use_lookup_key_value false
+  remove_json_key false
+</filter>
+```
+
+If following record is passed:
+
+```json
+{
+    "kubernetes_container_name" : "arbitrary",
+    "kubernetes_annotations_fluentd_org/keys" : "{\"nginx\": {\"es_type\": \"nginx\", \"es_index\": \"fluentd-nginx-\"}}"
+}
+```
+
+then the emitted record would be:
+
+```json
+{
+    "kubernetes_container_name" : "arbitrary",
+    "kubernetes_annotations_fluentd_org/keys" : "{\"nginx\": {\"es_type\": \"nginx\", \"es_index\": \"fluentd-nginx-\"}}",
+    "es_type": "nginx",
+    "es_index": "fluentd-nginx-"
+}
+```
+
+If the value of `lookup_key` is not present, not a JSON object, or is malformed
+JSON, the message will pass through unaltered (unless `remove_json_key` is set
+to true).
+
+## `json_key`
+
+The `json_key` specifies which log value to parse as json and perform a lookup
+in.
+
+## `lookup_key`
+
+The lookup key is used to specify the key within the deserialized `json_key`'s
+value.
+
+## `use_lookup_key_value`
+
+When `use_lookup_key_value` is set to true, the value of `lookup_key` is used,
+rather then the configuration key's name. Defaults to `true`.
+
+## `remove_json_key`
+
+When set to `true`, remove's the `json_key` from the message. Defaults to
+`false`.
+
+## License
+
+MIT (See [License](/LICENSE))

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new('test') do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task 'default' => ['build']

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-json-lookup.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'fluent-plugin-json-lookup'
+  gem.description = 'Filter plugin for looking up a json object out of a record'
+  gem.homepage    = 'https://github.com/skuid/fluent-plugin-json-lookup'
+  gem.summary     = gem.description
+  gem.version     = File.read('VERSION').strip
+  gem.authors     = ['Micah Hausler']
+  gem.email       = 'micah@skuid.com'
+  gem.has_rdoc    = false
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'fluentd', ['>= 0.10.58', '< 2']
+  gem.add_runtime_dependency 'yajl-ruby', '~> 1.0'
+  gem.add_development_dependency 'rake', '>= 0.9.2'
+  gem.add_development_dependency('test-unit', ['~> 3.1.4'])
+end

data/lib/fluent/plugin/filter_json_lookup.rb

@@ -0,0 +1,85 @@
+require 'fluent/filter'
+require 'yajl'
+
+module Fluent
+  if defined?(Filter)
+    class JsonLookupFilter < Filter
+      Fluent::Plugin.register_filter('json_lookup', self)
+
+      config_param :lookup_key,
+                   :string,
+                   default: nil,
+                   desc: <<-DESC
+The `lookup_key` is used in looking up the contents of the `json_key`.
+      DESC
+
+      config_param :use_lookup_key_value,
+                   :bool,
+                   default: true,
+                   desc: <<-DESC
+When `use_lookup_key_value` is set to `true`, the plugin performs the lookup
+using `lookup_key`'s value, rather than a hard-coded value.
+      DESC
+
+      config_param :json_key,
+                   :string,
+                   default: nil,
+                   desc: 'The `json_key`` is a key that contains a string of json.'
+
+      config_param :remove_json_key,
+                   :bool,
+                   default: false,
+                   desc: 'Remove the `json_key` from each record. Defaults to false.'
+
+      BUILTIN_CONFIGURATIONS = %w(
+        type @type log_level @log_level id @id lookup_key json_key
+        use_lookup_key_value remove_json_key
+      ).freeze
+
+      def configure(conf)
+        super
+
+        conf.each_pair do |k, v|
+          unless BUILTIN_CONFIGURATIONS.include?(k)
+            conf.key(k)
+            log.warn "Extra key provided! Ignoring '#{k} #{v}'"
+          end
+        end
+
+        # GC.start
+      end
+
+      def deserialize_and_lookup(content, lookup)
+        values = {}
+        begin
+          deserialized = Yajl.load(content)
+          if deserialized.is_a?(Hash) && deserialized.key?(lookup) && deserialized[lookup].is_a?(Hash)
+            values = deserialized[lookup]
+          end
+        rescue Yajl::ParseError
+          log.error "Error in plugin json_lookup, error parsing json_key's value #{content}'"
+        end
+        values
+      end
+
+      def filter_stream(_tag, es)
+        new_es = MultiEventStream.new
+        es.each do |time, record|
+          values = {}
+          if record.key?(@json_key)
+            lookup = @use_lookup_key_value ? record[@lookup_key] : @lookup_key
+            if record[@json_key][0] == '{'
+              values = deserialize_and_lookup(record[@json_key], lookup)
+            end
+          end
+          record.merge!(values)
+
+          record.delete(@json_key) if @remove_json_key && record.key?(@json_key)
+
+          new_es.add(time, record)
+        end
+        new_es
+      end
+    end
+  end
+end

data/test/test_filter_json_lookup.rb

@@ -0,0 +1,156 @@
+require 'fluent/test'
+require 'fluent/plugin/filter_json_lookup'
+require 'test/unit'
+
+exit unless defined?(Fluent::Filter)
+
+class JsonLookupFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @tag = 'test.tag'
+  end
+
+  CONFIG = '
+    @type json_lookup
+    lookup_key kubernetes_container_name
+    json_key kubernetes_annotations_fluentd_org/keys
+    remove_json_key true
+  '.freeze
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::FilterTestDriver.new(Fluent::JsonLookupFilter, @tag).configure(conf, true)
+  end
+
+  def test_lookup_exists
+    d = create_driver
+
+    d.run do
+      d.emit(
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"es_type": "nginx", "es_index": "fluentd-nginx-"}, "webapp": {"es_type": "application", "es_index": "fluentd-app-"}}'
+      )
+      d.emit(
+        'kubernetes_container_name' => 'webapp',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"es_type": "nginx", "es_index": "fluentd-nginx-"}, "webapp": {"es_type": "application", "es_index": "fluentd-app-"}}'
+      )
+    end
+
+    assert_equal [
+      { 'kubernetes_container_name' => 'nginx', 'es_type' => 'nginx', 'es_index' => 'fluentd-nginx-' },
+      { 'kubernetes_container_name' => 'webapp', 'es_type' => 'application', 'es_index' => 'fluentd-app-' }
+    ], d.filtered_as_array.map(&:last)
+  end
+
+  def test_lookup_exists_keep_key
+    d = create_driver '
+      @type json_lookup
+      lookup_key kubernetes_container_name
+      json_key kubernetes_annotations_fluentd_org/keys
+    '
+
+    d.run do
+      d.emit(
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}, "webapp": {"t": "application", "i": "fluentd-app-"}}'
+      )
+      d.emit(
+        'kubernetes_container_name' => 'webapp',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}, "webapp": {"t": "application", "i": "fluentd-app-"}}'
+      )
+    end
+
+    assert_equal [
+      {
+        'i' => 'fluentd-nginx-',
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}, "webapp": {"t": "application", "i": "fluentd-app-"}}',
+        't' => 'nginx'
+      },
+      {
+        'i' => 'fluentd-app-',
+        'kubernetes_container_name' => 'webapp',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}, "webapp": {"t": "application", "i": "fluentd-app-"}}',
+        't' => 'application'
+      }
+    ], d.filtered_as_array.map(&:last)
+  end
+
+  def test_lookup_exists_hard_code
+    d = create_driver '
+      @type json_lookup
+      lookup_key hardcoded
+      json_key kubernetes_annotations_fluentd_org/keys
+      use_lookup_key_value false
+      remove_json_key true
+    '
+
+    d.run do
+      d.emit(
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{"hardcoded": {"t": "hardcoded", "i": "fluentd-hardcoded-"}, "webapp": {"t": "application", "i": "fluentd-app-"}}'
+      )
+      d.emit(
+        'kubernetes_container_name' => 'webapp',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}, "webapp": {"t": "application", "i": "fluentd-app-"}}'
+      )
+    end
+
+    assert_equal [
+      {
+        'i' => 'fluentd-hardcoded-',
+        'kubernetes_container_name' => 'nginx',
+        't' => 'hardcoded'
+      },
+      {
+        'kubernetes_container_name' => 'webapp'
+      }
+    ], d.filtered_as_array.map(&:last)
+  end
+
+  def test_lookup_absent
+    d = create_driver
+
+    d.run do
+      d.emit(
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{"webapp": {"t": "application", "i": "fluentd-app-"}}'
+      )
+      d.emit(
+        'kubernetes_container_name' => 'webapp',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}}'
+      )
+      d.emit(
+        'other_key' => 'value',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": {"t": "nginx", "i": "fluentd-nginx-"}}'
+      )
+      d.emit('kubernetes_container_name' => 'webapp')
+    end
+
+    assert_equal [
+      { 'kubernetes_container_name' => 'nginx' },
+      { 'kubernetes_container_name' => 'webapp' },
+      { 'other_key' => 'value' },
+      { 'kubernetes_container_name' => 'webapp' }
+    ], d.filtered_as_array.map(&:last)
+  end
+
+  def test_lookup_malformed
+    d = create_driver
+
+    d.run do
+      d.emit(
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{{"webapp": {"t": "application", "i": "fluentd-app-"}}'
+      )
+      d.emit(
+        'kubernetes_container_name' => 'nginx',
+        'kubernetes_annotations_fluentd_org/keys' => '{"nginx": ["t", "nginx", "i", "fluentd-nginx-"]}'
+      )
+    end
+
+    assert_equal [
+      { 'kubernetes_container_name' => 'nginx' },
+      { 'kubernetes_container_name' => 'nginx' }
+    ], d.filtered_as_array.map(&:last)
+  end
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-json-lookup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Micah Hausler
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.4
+description: Filter plugin for looking up a json object out of a record
+email: micah@skuid.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.v0.12
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-json-lookup.gemspec
+- lib/fluent/plugin/filter_json_lookup.rb
+- test/test_filter_json_lookup.rb
+homepage: https://github.com/skuid/fluent-plugin-json-lookup
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Filter plugin for looking up a json object out of a record
+test_files:
+- test/test_filter_json_lookup.rb