fluent-plugin-jq 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fef74bd93979266e240f96325d7686b004acb091e16264713e5ccb4431a632d
-  data.tar.gz: f7cc5b63c3863efd09a472a4d0f48c62e3879df6f9eb1ab095bee309f7dfe7e0
+  metadata.gz: 7bee9a7bb5bd22f8b9a08cc91e3e0b8c82cc26c3a22c79840513c36d18eeda0b
+  data.tar.gz: adaa5a025cff8142f3e34f9a518df08dcab30b593e94193ca046a902ca94dc84
 SHA512:
-  metadata.gz: 3e6335fd5b5099d5165182a28b889a7d7df007466f7c84ac1da31a0c709be0ff4a55baef6125b1351162a3174365140b22a47c110d42ba5263953693eced5179
-  data.tar.gz: df42a0cec7562e2a5882666923169a6871ea046c8c79a28ae9f45cd089f62f30498837948ca9801f162618c53daa5c6d741626642f6a858f3f9e1cdd0d6f555e
+  metadata.gz: c7d787d80c6b742ad54e74d6fea4273710f59308b1629cf6064d06a5e4ad397d32bcf97c9de12b5045221555b60537ff859e171c25a5c2c5df15c83e07a04e87
+  data.tar.gz: ef43d2ed72e2c4d0a4f5fd0559cd1e327f3bdb52dcaaa8f9f762a3081dd22642354ba5f74e2eaac5858c2da40ce5a803902e53a489f6503ef9ee9b2f3a054283

data/Gemfile.lock

@@ -1,10 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jq (0.4.0)
+    fluent-plugin-jq (0.5.0)
       fluentd (>= 0.14.10, < 2)
       multi_json (~> 1.13)
-      ruby-jq (~> 0.1)
 
 GEM
   remote: https://rubygems.org/
@@ -35,8 +34,6 @@ GEM
     multi_json (1.13.1)
     power_assert (1.1.1)
     rake (12.3.0)
-    ruby-jq (0.1.7)
-      multi_json
     serverengine (2.0.6)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)

data/README.md

@@ -13,7 +13,7 @@ A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedola
 
 See also: [Plugin Management](https://docs.fluentd.org/v1.0/articles/plugin-management).
 
-Before you install this plugin, please make sure that `libjq` has been installed on your machine. For example, it's called `jq-dev` on alpine linux; while on debian, and ubuntu, you will find it as the `libjq-dev` package.
+Before you install this plugin, please make sure the `jq` command line tool has been installed on your machine. Plugins defined in this gem will call the `jq` command to make the transformation.
 
 ### RubyGems
 
@@ -73,7 +73,7 @@ This must be `jq`.
 
 ##### jq (string) (required)
 
-The jq filter for formatting income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used.
+The jq filter for formatting income events. The returned result should be a string, if not, it will be encoded to a JSON representation in a string.
 
 ##### on_error (enum) (optional)
 

data/example/Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="Zhimin (Gimi) Liang (https://github.com/Gimi)"
 COPY . /tmp/fluent-plugin-jq/
 
 RUN apk update \
- && apk add --no-cache jq-dev \
+ && apk add --no-cache jq \
  && apk add --no-cache --virtual .build-deps \
         build-base \
         git \

data/fluent-plugin-jq.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jq"
-  spec.version = "0.4.0"
+  spec.version = "0.5.0"
   spec.authors = ["Zhimin (Gimi) Liang"]
   spec.email   = ["liang.gimi@gmail.com"]
 
@@ -24,6 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "coveralls", "~> 0.8"
 
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
-  spec.add_runtime_dependency "ruby-jq", "~> 0.1"
   spec.add_runtime_dependency "multi_json", "~> 1.13"
 end

data/lib/fluent/plugin/filter_jq_transformer.rb

@@ -15,39 +15,25 @@
 # limitations under the License.
 
 require "fluent/plugin/filter"
+require "fluent/plugin/jq_mixin"
 
 module Fluent
   module Plugin
     class JqTransformerFilter < Fluent::Plugin::Filter
       Fluent::Plugin.register_filter("jq_transformer", self)
 
-      desc 'The jq filter used to transform the input. The result of the filter should return an object.'
-      config_param :jq, :string
+      include JqMixin
 
-      def initialize
-	super
-	require "jq"
-      end
-
-      def configure(conf)
-	super
-	@jq_filter = JQ::Core.new @jq
-      rescue JQ::Error
-	raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
-      end
+      config_set_desc :jq, 'The jq filter used to transform the input. The result of the filter should return an object.'
 
       def filter(tag, time, record)
-	new_record = [].tap { |buf|
-	  @jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
-	    buf << MultiJson.load("[#{r}]").first
-	  }
-	}.first
+	new_record = jq_transform tag: tag, time: time, record: record
 	return new_record if new_record.is_a?(Hash)
 
 	log.error "jq filter #{@jq} did not return a hash, skip this record."
 	nil
-      rescue JQ::Error
-	log.error "Failed to transform #{MultiJson.dump record} with #{@jq}, error: #{$!.message}"
+      rescue JqError
+	log.error "Filter failed with #{@jq}#{log.on_debug { ' on ' + MultiJson.dump(tag: tag, time: time, record: record) }}, error: #{$!.message}"
 	nil
       end
     end

data/lib/fluent/plugin/formatter_jq.rb

@@ -15,53 +15,39 @@
 # limitations under the License.
 
 require "fluent/plugin/formatter"
+require "fluent/plugin/jq_mixin"
 
 module Fluent
   module Plugin
     class JqFormatter < Fluent::Plugin::Formatter
       Fluent::Plugin.register_formatter("jq", self)
 
-      desc 'The jq program used to format income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used.'
-      config_param :jq, :string, default: nil
+      include JqMixin
 
-      desc 'The jq program used to format income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used. DEPRECATED.'
-      config_param :jq_program, :string, deprecated: 'use jq instead.', default: nil
+      config_set_desc :jq, 'The jq filter used to format income events. If the result returned from the filter is not a string, it will be encoded as a JSON string.'
 
       desc 'Defines the behavior on error happens when formatting an event. "skip" will skip the event; "ignore" will ignore the error and return the JSON representation of the original event; "raise_error" will raise a RuntimeError.'
       config_param :on_error, :enum, list: [:skip, :ignore, :raise_error], default: :ignore
 
       def initialize
 	super
-	require "jq"
-      end
-
-      def configure(conf)
-	super
-
-	@jq = @jq_program unless @jq
-	raise Fluent::ConfigError, "jq is required." unless @jq
-
-	@jq_filter = JQ::Core.new @jq
-      rescue JQ::Error
-	raise Fluent::ConfigError, "Could not parse jq filter #{@jq}, error: #{$!.message}"
       end
 
       def format(tag, time, record)
-	item = [].tap { |buf|
-	  @jq_filter.update(MultiJson.dump(record), false) { |r|
-	    buf << MultiJson.load("[#{r}]").first
-	  }
-	}.first
-	return item if item.instance_of?(String)
-	MultiJson.dump item
-      rescue JQ::Error
-	msg = "Failed to format #{record.to_json} with #{@jq}, error: #{$!.message}"
+	item = jq_transform record
+	if item.instance_of?(String)
+	  item
+	else
+	  MultiJson.dump item
+	end
+      rescue JqError
+	msg = "Format failed with #{@jq}#{log.on_debug { ' on ' + MultiJson.dump(record) }}, error: #{$!.message}"
 	log.error msg
 	case @on_error
 	when :skip
 	  return ''
 	when :ignore
-	  return record.to_json
+	  return MultiJson.dump(record)
 	when :raise_error
 	  raise msg
 	end

data/lib/fluent/plugin/jq_mixin.rb

@@ -0,0 +1,47 @@
+require 'shellwords'
+require 'multi_json'
+
+module JqMixin
+  JqError = Class.new(RuntimeError)
+
+  def self.included(plugin)
+    plugin.config_param :jq, :string
+  end
+
+  def configure(conf)
+    super
+    p = start_process(null_input: true)
+    err = p.read
+    raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{err}" if err =~ /compile error/m
+  rescue
+    raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
+  ensure
+    p.close if p # if `super` fails, `p` will be `nil`
+  end
+
+  def start
+    super
+    @jq_process = start_process
+  end
+
+  def shutdown
+    @jq_process.close rescue nil
+    super
+  end
+
+  def start_process(filter: @jq, null_input: false)
+    IO.popen(%Q"jq #{'-n' if null_input} --unbuffered -c '#{filter}' 2>&1", 'r+')
+  end
+
+  def jq_transform(object)
+    @jq_process.puts MultiJson.dump(object)
+    result = @jq_process.gets
+    MultiJson.load result
+  rescue MultiJson::ParseError
+    raise JqError.new(result)
+  rescue Errno::EPIPE
+    @jq_process.close
+    @jq_process = start_process
+    retry
+  end
+end

data/lib/fluent/plugin/out_jq.rb

@@ -15,30 +15,20 @@
 # limitations under the License.
 
 require 'fluent/plugin/output'
+require 'fluent/plugin/jq_mixin'
 
 module Fluent::Plugin
   class JqOutput < Output
     Fluent::Plugin.register_output('jq', self)
     helpers :event_emitter
 
-    desc 'The jq filter used to transform the input. The result of the filter should return an object.'
-    config_param :jq, :string
+    include JqMixin
+
+    config_set_desc :jq, 'The jq filter used to transform the input. If the filter returns an array, each object in the array will be a new record.'
 
     desc 'The prefix to be removed from the input tag when outputting a new record.'
     config_param :remove_tag_prefix, :string, default: ''
 
-    def initialize
-      super
-      require "jq"
-    end
-
-    def configure(conf)
-      super
-      @jq_filter = JQ::Core.new @jq
-    rescue JQ::Error
-      raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
-    end
-
     def multi_workers_ready?
       true
     end
@@ -47,14 +37,11 @@ module Fluent::Plugin
       new_es = Fluent::MultiEventStream.new
       es.each do |time, record|
 	begin
-	  @jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
-	    # the filter could return an array
-	    new_records = [MultiJson.load("[#{r}]").first]
-	    new_records.flatten!
-	    new_records.each { |new_record| new_es.add time, new_record }
-	  }
-	rescue JQ::Error
-	  log.error "Failed to transform #{MultiJson.dump record} with #{@jq}, error: #{$!.message}"
+	  new_records = jq_transform tag: tag, time: time, record: record
+	  new_records = [new_records] unless new_records.is_a?(Array)
+	  new_records.each { |new_record| new_es.add time, new_record }
+	rescue JqError
+	  log.error "Process failed with #{@jq}#{log.on_debug {' on ' + MultiJson.dump(record)}}, error: #{$!.message}"
 	end
       end
 

data/lib/fluent/plugin/parser_jq.rb

@@ -15,40 +15,26 @@
 # limitations under the License.
 
 require "fluent/plugin/parser"
+require 'fluent/plugin/jq_mixin'
 
 module Fluent
   module Plugin
     class JqParser < Fluent::Plugin::Parser
       Fluent::Plugin.register_parser("jq", self)
 
-      desc 'The jq filter used to format the input. The result of the filter must return an object.'
-      config_param :jq, :string
+      include JqMixin
 
-      def initialize
-	super
-	require "jq"
-      end
-
-      def configure(conf)
-	super
-	@jq_filter = JQ::Core.new @jq
-      rescue JQ::Error
-	raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
-      end
+      config_set_desc :jq, 'The jq filter used to format the input. The result of the filter must return an object.'
 
       def parse(text)
-	record = [].tap { |buf|
-	  @jq_filter.update(MultiJson.dump(text), false) { |r|
-	    buf << MultiJson.load("[#{r}]").first
-	  }
-	}.first
+	record = jq_transform text
 	if record.is_a?(Hash)
 	  yield parse_time(record), record
 	else
 	  log.error "jq filter #{@jq} did not return a hash, skip this record."
 	end
-      rescue JQ::Error
-	log.error "Failed to parse #{text} with #{@jq}, error: #{$!.message}"
+      rescue JqError
+	log.error "Parse failed with #{@jq}#{log.on_debug {' on ' + text}}, error: #{$!.message}"
 	nil
       end
     end

data/test/plugin/test_filter_jq_transformer.rb

@@ -9,6 +9,10 @@ class JqTransformerFilterTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  teardown do
+    @driver.instance.shutdown if @driver
+  end
+
   test "it should require jq" do
     assert_raise(Fluent::ConfigError) { create_driver '' }
   end
@@ -65,6 +69,6 @@ class JqTransformerFilterTest < Test::Unit::TestCase
   private
 
   def create_driver(conf)
-    Fluent::Test::Driver::Filter.new(Fluent::Plugin::JqTransformerFilter).configure(conf)
+    @driver = Fluent::Test::Driver::Filter.new(Fluent::Plugin::JqTransformerFilter).configure(conf).tap { |d| d.instance.start }
   end
 end

data/test/plugin/test_formatter_jq.rb

@@ -7,6 +7,10 @@ class JqFormatterTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  teardown do
+    @driver.instance.shutdown if @driver
+  end
+
   test "it should require jq parameter" do
     assert_raise(Fluent::ConfigError) { create_driver '' }
   end
@@ -48,6 +52,6 @@ class JqFormatterTest < Test::Unit::TestCase
   private
 
   def create_driver(conf)
-    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::JqFormatter).configure(conf)
+    @driver = Fluent::Test::Driver::Formatter.new(Fluent::Plugin::JqFormatter).configure(conf).tap { |d| d.instance.start }
   end
 end

data/test/plugin/test_parser_jq.rb

@@ -9,6 +9,10 @@ class JqParserTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  teardown do
+    @driver.instance.shutdown if @driver
+  end
+
   test "it should require jq" do
     assert_raise(Fluent::ConfigError) { create_driver '' }
   end
@@ -39,6 +43,6 @@ class JqParserTest < Test::Unit::TestCase
   private
 
   def create_driver(conf)
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::JqParser).configure(conf)
+    @driver = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JqParser).configure(conf).tap { |d| d.instance.start }
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jq
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Zhimin (Gimi) Liang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-28 00:00:00.000000000 Z
+date: 2018-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,20 +86,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: ruby-jq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -135,6 +121,7 @@ files:
 - fluent-plugin-jq.gemspec
 - lib/fluent/plugin/filter_jq_transformer.rb
 - lib/fluent/plugin/formatter_jq.rb
+- lib/fluent/plugin/jq_mixin.rb
 - lib/fluent/plugin/out_jq.rb
 - lib/fluent/plugin/parser_jq.rb
 - run_ci.sh