fluent-plugin-json-in-json-2 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e99746f88495ff8ed13fd6d295a4522cc641172c
+  data.tar.gz: 870714d1b589915f6642de346c7ca21f19f7a0cb
+SHA512:
+  metadata.gz: f0ebd7c9647c3e36afa83c732929339c8ddab29b5d02b592799e76687024842f5f51c15cc29640a2a4397577290c128f8bbfe798d087a7e25a40ae2f5f51f38b
+  data.tar.gz: 4ee7b4768620d0179f60e63776b631207b63650d766de565a689ead87ac75a8deda9ed913f2b0f234a29f05ef98a0bb42abf4437b63b4f9339bcffddd78c81e6

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-json-in-json.gemspec
+gemspec

data/README.md

@@ -0,0 +1,49 @@
+# fluent-plugin-json-in-json
+
+## Requirements
+
+| fluent-plugin-json-in-json| Fluentd     | Ruby   |
+|---------------------------|-------------|--------|
+| >= 1.0.0                  | >= v0.14.0  | >= 2.1 |
+|  < 1.0.0                  | >= v0.12.0  | >= 1.9 |
+
+This fluentd parser plugin parses JSON log lines with nested JSON strings. For
+example, given a docker log of ``{"log": "{\"foo\": \"bar\"}"}``, the log record
+will be parsed into ``{:log => { :foo => "bar" }}``.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-json-in-json'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-json-in-json
+
+
+## Usage
+
+```
+<source>
+  type tail
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /var/log/fluentd-docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.*
+  format json_in_json
+  read_from_head true
+</source>
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+  t.warning = true
+  t.ruby_opts = ['-Eascii-8bit:ascii-8bit']
+end

data/fluent-plugin-json-in-json.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-json-in-json-2"
+  spec.version       = "1.0.0"
+  spec.authors       = ["Gavin M. Roy", "Arcadiy Ivanov", "Alik Khilazhev"]
+  spec.email         = ["gavinmroy@gmail.com", "arcadiy@ivanov.biz", "alikhil@mail.ru"]
+  spec.description   = %q{Parser plugin that parses JSON attributes with JSON strings in them}
+  spec.summary       = %q{Parser plugin that parses JSON attributes with JSON strings in them}
+  spec.homepage      = "https://github.com/arcivanov/fluent-plugin-json-in-json"
+  spec.license       = "BSD"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
+  spec.add_runtime_dependency 'yajl-ruby', '~> 1.0'
+
+  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+
+  spec.add_development_dependency 'test-unit', ['~> 3.2']
+  spec.add_development_dependency 'test-unit-rr', ['~> 1.0']
+end

data/lib/fluent/plugin/parser_json_in_json.rb

@@ -0,0 +1,46 @@
+require 'fluent/parser'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class JSONInJSONParser < Parser
+      Plugin.register_parser('json_in_json', self)
+
+      config_set_default :time_key, 'time'
+      config_set_default :time_type, :float
+
+      def configure(conf)
+        if conf.has_key?('time_format')
+          conf['time_type'] ||= 'string'
+        end
+
+        super
+      end
+
+      def parse(text)
+        record = Yajl.load(text)
+
+        values = Hash.new
+
+        record.each do |k, v|
+          if v.is_a?(String) && /^\s*(\{|\[)/ =~ v
+            deserialized = Yajl.load(v)
+            if deserialized.is_a?(Hash)
+              values.merge!(deserialized)
+              record.delete k
+            elsif deserialized.is_a?(Array)
+              values[k] = deserialized
+            end
+          end
+        end
+        record.merge!(values)
+
+        time, record = convert_values(parse_time(record), record)
+
+        yield time, record
+      rescue Yajl::ParseError
+        yield nil, nil
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,128 @@
+# Some tests use Hash instead of Element for configure.
+# We should rewrite these tests in the future and remove this ad-hoc code
+class Hash
+  def corresponding_proxies
+    @corresponding_proxies ||= []
+  end
+
+  def to_masked_element
+    self
+  end
+end
+
+require 'rr'
+require 'test/unit'
+require 'test/unit/rr'
+require 'fileutils'
+require 'fluent/config/element'
+require 'fluent/log'
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+require 'fluent/msgpack_factory'
+require 'fluent/time'
+require 'serverengine'
+
+module Fluent
+  module Plugin
+    class TestBase < Base
+      # a base plugin class, but not input nor output
+      # mainly for helpers and owned plugins
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+    end
+  end
+end
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+include Fluent::Test::Helpers
+
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
+  case protocol
+  when :tcp
+    unused_port_tcp(num)
+  when :udp
+    unused_port_udp(num, bind: bind)
+  else
+    raise ArgumentError, "unknown protocol: #{protocol}"
+  end
+end
+
+def unused_port_tcp(num = 1)
+  ports = []
+  sockets = []
+  num.times do
+    s = TCPServer.open(0)
+    sockets << s
+    ports << s.addr[1]
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+PORT_RANGE_AVAILABLE = (1024...65535)
+
+def unused_port_udp(num = 1, bind: "0.0.0.0")
+  family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+  ports = []
+  sockets = []
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
+    u = UDPSocket.new(family)
+    if (u.bind(bind, port) rescue nil)
+      ports << port
+      sockets << u
+    else
+      u.close
+    end
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+def waiting(seconds, logs: nil, plugin: nil)
+  begin
+    Timeout.timeout(seconds) do
+      yield
+    end
+  rescue Timeout::Error
+    if logs
+      STDERR.print(*logs)
+    elsif plugin
+      STDERR.print(*plugin.log.out.logs)
+    end
+    raise
+  end
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
+
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+logdev = Fluent::Test::DummyLogDevice.new
+logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+$log ||= Fluent::Log.new(logger)

data/test/test_parser.rb

@@ -0,0 +1,54 @@
+require_relative 'helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_json_in_json'
+
+class JsonInJsonParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONInJSONParser)
+  end
+
+  def test_parse_float_time()
+    @parser.configure({})
+    @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT","log_json":" {\\"field1\\":\\"field1 value\\",\\"field2\\":40}"}') { |time, record|
+      assert_equal(event_time('2013-02-28 12:00:00 +0900').to_f, time.to_f)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'size'   => 777,
+                     'method' => 'PUT',
+                     'field1' => 'field1 value',
+                     'field2' => 40
+                   }, record)
+    }
+    @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT","log_json":"{\\"field1\\":"}') { |time, record|
+      assert_equal(event_time('2013-02-28 12:00:00 +0900').to_f, time.to_f)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'size'   => 777,
+                     'method' => 'PUT',
+                     'log_json' => '{"field1":'
+                   }, record)
+    }
+    @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT","log_json":"[\\"val1\\",\\"val2\\"]"') { |time, record|
+      assert_equal(event_time('2013-02-28 12:00:00 +0900').to_f, time.to_f)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'size'   => 777,
+                     'method' => 'PUT',
+                     'log_json' => ['val1', 'val2']
+                   }, record)
+    }
+  end
+
+  def test_parse_string_time()
+    @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%S.%NZ', 'keep_time_key' => 'true')
+    @parser.instance.parse('{"log":"2018-06-26 13:20:44.075  INFO --- [pool-8-thread-3] outgoing","stream":"stdout","time":"2018-06-26T13:20:44.076022960Z"}') { |time, record|
+      assert_equal(event_time('2018-06-26 13:20:44.076022960 -0400').to_f, time.to_f)
+      assert_equal({
+                    'log'=>'2018-06-26 13:20:44.075  INFO --- [pool-8-thread-3] outgoing',
+                    'stream'=>'stdout',
+                    'time'=>'2018-06-26T13:20:44.076022960Z'
+                   }, record)
+    }
+  end
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-json-in-json-2
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Gavin M. Roy
+- Arcadiy Ivanov
+- Alik Khilazhev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-06-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Parser plugin that parses JSON attributes with JSON strings in them
+email:
+- gavinmroy@gmail.com
+- arcadiy@ivanov.biz
+- alikhil@mail.ru
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-json-in-json.gemspec
+- lib/fluent/plugin/parser_json_in_json.rb
+- test/helper.rb
+- test/test_parser.rb
+homepage: https://github.com/arcivanov/fluent-plugin-json-in-json
+licenses:
+- BSD
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Parser plugin that parses JSON attributes with JSON strings in them
+test_files:
+- test/helper.rb
+- test/test_parser.rb