fluent-plugin-jq 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24f79462819d358078a156e8ea375770075384849193ceff156938a397c9ef34
-  data.tar.gz: acdb2479ecd7c5a5d2d8baededdb5c11895454628ff1eb2bd2febf5363d64505
+  metadata.gz: 85add62e095656dfaeb43bff27ae998d7ff39cd523a2f0d2d564d981a8e3ccf6
+  data.tar.gz: ec02443a917e479f171ecc44603f39b4f3c7f5e3ecbe629c0a2c4ce8eabe2bc4
 SHA512:
-  metadata.gz: 454db6f7ed58a47f1c4b6fa280793b06b4b8d6e898e3905aba79fb6fa184787f40f9bd475284e9e7d8ee26cf8cf1c13da2d96dc11465327f732cd6f35207f090
-  data.tar.gz: 7b6d295570fe9f509e0534e2ac2bc57c3cedf6e81b86fffc1960cf814f1e16e27c3d66d3418c7924f7c3c71af5eeba6e67b87f7587956ea82f059a4bac9b1ff1
+  metadata.gz: 4f340a9588d62e49758f7e8331768d912b4518876ef97945a82f9a6977fd516b23a32ece0712371f42bbd29607961e1ceb9ce2b0ce95daf8ba832d3a727ba247
+  data.tar.gz: 0601f3c89a55354fd844c6ef0152879d020a0af3c4d9980c9e5109622f25affd54ace34fe351f44e50eb825af137861721d90c015c74b19f97847c44b5341298

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jq (0.2.0)
+    fluent-plugin-jq (0.3.0)
       fluentd (>= 0.14.10, < 2)
       multi_json (~> 1.13)
       ruby-jq (~> 0.1)

data/README.md

@@ -3,9 +3,10 @@
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-jq.svg)](https://badge.fury.io/rb/fluent-plugin-jq)
 [![Build Status](https://travis-ci.org/Gimi/fluent-plugin-jq.svg?branch=master)](https://travis-ci.org/Gimi/fluent-plugin-jq)
 
-A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains two plugins:
+A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains three plugins:
 * `jq` formatter - a formatter plugin formats inputs using jq filters.
 * `jq_transformer` - a filter plugin transform inputs.
+* `jq` output - a output plugin uses jq filter to generate new events.
 
 ## Installation
 
@@ -114,6 +115,48 @@ This must be `jq_transformer`.
 
 The jq filter used to transform the input. The result of the filter should return an object. If after applying the transforming the new event is not an object (a hash), the event will be dropped.
 
+### `jq` Output
+
+#### Example
+
+```
+<match raw.data>
+  @type jq
+  jq .record | to_entries
+  remove_tag_prefix raw
+</filter>
+```
+
+The above example will generate one event for each key-value pair in each input, and then tag it with "data" ("raw." is removed), and send it back to router. For example, given an input like
+
+```javascript
+{"logLevel": "info", "log": "this is an example."}
+```
+
+It generates two new events:
+
+```javascript
+{"key": "logLevel", "value": "info"}
+{"key": log", "value": "this is an example."}
+```
+
+#### Parameters
+
+##### @type (string) (required)
+
+This must be `jq`.
+
+##### jq (string) (required)
+
+The jq filter used to generate new events. The result of the filter should return an object or an array of objects. New events will be put back to router so that they will be processed again.
+
+##### remove_tag_prefix (string) (optional)
+
+The prefix to remove from the input tag when outputting a new event. A prefix has to be a complete tag part.
+Example: If `remove_tag_prefix` is set to 'foo', the input tag foo.bar.baz is transformed to bar.baz and the input tag 'foofoo.bar' is not modified.
+
+Default value: `""`.
+
 ### Built-in Example
 
 Once you clone the project from github, you can run the following commands to see a real example for the plugins.

data/example/fluent.conf

@@ -1,9 +1,15 @@
 <source>
   @type dummy
-  tag dummy
+  tag raw.dummy
   dummy {"log": "everything goes well", "stream": "stdout"}
 </source>
 
+<match raw.dummy>
+  @type jq
+  jq .record | to_entries
+  remove_tag_prefix raw
+</match>
+
 <filter dummy>
   @type jq_transformer
   jq .record + {tag, time}
@@ -13,6 +19,6 @@
   @type stdout
   <format>
     @type jq
-    jq '"\(.time | todate) [\(.tag)] \(.stream) => \(.log)"'
+    jq '"\(.time | todate) [\(.tag)] \(.key) => \(.value)"'
   </format>
 </match>

data/fluent-plugin-jq.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jq"
-  spec.version = "0.2.0"
+  spec.version = "0.3.0"
   spec.authors = ["Zhimin (Gimi) Liang"]
   spec.email   = ["liang.gimi@gmail.com"]
 

data/lib/fluent/plugin/filter_jq_transformer.rb

@@ -31,13 +31,17 @@ module Fluent
 
       def configure(conf)
 	super
-	JQ::Core.new @jq
+	@jq_filter = JQ::Core.new @jq
       rescue JQ::Error
 	raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
       end
 
       def filter(tag, time, record)
-	new_record = JQ(MultiJson.dump(tag: tag, time: time, record: record)).search(@jq).first
+	new_record = [].tap { |buf|
+	  @jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
+	    buf << MultiJson.load("[#{r}]").first
+	  }
+	}.first
 	return new_record if new_record.is_a?(Hash)
 
 	log.error "jq filter #{@jq} did not return a hash, skip this record."

data/lib/fluent/plugin/formatter_jq.rb

@@ -41,13 +41,17 @@ module Fluent
 	@jq = @jq_program unless @jq
 	raise Fluent::ConfigError, "jq is required." unless @jq
 
-	JQ::Core.new @jq
+	@jq_filter = JQ::Core.new @jq
       rescue JQ::Error
 	raise Fluent::ConfigError, "Could not parse jq filter #{@jq}, error: #{$!.message}"
       end
 
       def format(tag, time, record)
-	item = JQ(MultiJson.dump(record)).search(@jq).first
+	item = [].tap { |buf|
+	  @jq_filter.update(MultiJson.dump(record), false) { |r|
+	    buf << MultiJson.load("[#{r}]").first
+	  }
+	}.first
 	return item if item.instance_of?(String)
 	MultiJson.dump item
       rescue JQ::Error

data/lib/fluent/plugin/out_jq.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class JqOutput < Output
+    Fluent::Plugin.register_output('jq', self)
+    helpers :event_emitter
+
+    desc 'The jq filter used to transform the input. The result of the filter should return an object.'
+    config_param :jq, :string
+
+    desc 'The prefix to be removed from the input tag when outputting a new record.'
+    config_param :remove_tag_prefix, :string, default: ''
+
+    def initialize
+      super
+      require "jq"
+    end
+
+    def configure(conf)
+      super
+      @jq_filter = JQ::Core.new @jq
+    rescue JQ::Error
+      raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def process(tag, es)
+      new_es = Fluent::MultiEventStream.new
+      es.each do |time, record|
+	begin
+	  @jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
+	    # the filter could return an array
+	    new_records = [MultiJson.load("[#{r}]").first]
+	    new_records.flatten!
+	    new_records.each { |new_record| new_es.add time, new_record }
+	  }
+	rescue JQ::Error
+	  log.error "Failed to transform #{MultiJson.dump record} with #{@jq}, error: #{$!.message}"
+	end
+      end
+
+      new_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+      router.emit_stream(new_tag, new_es)
+    end
+  end
+end

data/test/plugin/test_out_jq.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "helper"
+require "fluent/test/driver/output"
+require "fluent/plugin/out_jq"
+
+class JqOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "it should require jq" do
+    assert_raise(Fluent::ConfigError) { create_driver '' }
+  end
+
+  test "it should raise error on invalid jq program" do
+    e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
+    assert_match(/compile error/, e.message)
+  end
+
+  test "it should work on tag" do
+    events = create_driver('jq "{tag}"').events
+    assert_equal events.size, 1
+    assert_equal events[0][2]["tag"], "some.tag"
+  end
+
+  test "it should work on time" do
+    events = create_driver('jq "{time: .time | gmtime }"').events
+    assert_equal events.size, 1
+    assert_equal events[0][2]["time"][0..5], [2018, 2, 22, 1, 23, 45]
+  end
+
+  test "it should support array" do
+    events = create_driver('jq ".record | to_entries"').events.map { |evt| evt[2] }
+    assert_equal events.size, 2
+    assert_include events, {"key" => "log", "value" => "this is a log"}
+    assert_include events, {"key" => "source", "value" => "stdout"}
+  end
+
+  test "it should remove specified tag prefix" do
+    events = create_driver(<<~CONF).events
+      jq ".record"
+      remove_tag_prefix some
+    CONF
+
+    assert_equal events.size, 1
+    assert_equal events[0][0], "tag"
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::JqOutput).configure(conf).tap { |d|
+      time = event_time("2018-03-22 01:23:45 UTC")
+      d.run { d.feed "some.tag", time, {"log" => "this is a log", "source" => "stdout"} }
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jq
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Zhimin (Gimi) Liang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-02 00:00:00.000000000 Z
+date: 2018-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -134,10 +134,12 @@ files:
 - fluent-plugin-jq.gemspec
 - lib/fluent/plugin/filter_jq_transformer.rb
 - lib/fluent/plugin/formatter_jq.rb
+- lib/fluent/plugin/out_jq.rb
 - run_ci.sh
 - test/helper.rb
 - test/plugin/test_filter_jq_transformer.rb
 - test/plugin/test_formatter_jq.rb
+- test/plugin/test_out_jq.rb
 homepage: https://github.com/Gimi/fluent-plugin-jq
 licenses:
 - Apache-2.0
@@ -165,4 +167,5 @@ summary: Fluentd plugins uses the jq engine.
 test_files:
 - test/helper.rb
 - test/plugin/test_filter_jq_transformer.rb
+- test/plugin/test_out_jq.rb
 - test/plugin/test_formatter_jq.rb