fluent-plugin-jq 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16f6b69c379ba56689e84e98da0a1b4d229588b14da0f9b600b57d519255d214
-  data.tar.gz: 96050f9bd7ccccdbed4b2d912c40f3d2188d66728cf3aeb33a2e2eb31d840eab
+  metadata.gz: 24f79462819d358078a156e8ea375770075384849193ceff156938a397c9ef34
+  data.tar.gz: acdb2479ecd7c5a5d2d8baededdb5c11895454628ff1eb2bd2febf5363d64505
 SHA512:
-  metadata.gz: f74a75908cc59e7153da6bfe96a5e4b79e2cde29b0e191ec630da2569c0ab385d13c7000fe29bdddc12f765cb74c915da26af01cea2c10e3fb5f033f822b0a0b
-  data.tar.gz: aa1c37f3139b90372c11b5e8c95e5151ecc48526bea2b0e43e35769b536978483f3c6680432fcf4461a75f675d33a41653d055ae81be028c553511a5a3dc3a35
+  metadata.gz: 454db6f7ed58a47f1c4b6fa280793b06b4b8d6e898e3905aba79fb6fa184787f40f9bd475284e9e7d8ee26cf8cf1c13da2d96dc11465327f732cd6f35207f090
+  data.tar.gz: 7b6d295570fe9f509e0534e2ac2bc57c3cedf6e81b86fffc1960cf814f1e16e27c3d66d3418c7924f7c3c71af5eeba6e67b87f7587956ea82f059a4bac9b1ff1

data/CHANGES.md

@@ -0,0 +1,4 @@
+# v0.2.0
+* Added "jq_transformer" filter plugin.
+* Introduced "multi\_json".
+* Rename "jq_program" to "jq" for jq formatter.

data/Gemfile.lock

@@ -1,15 +1,23 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jq (0.1.0)
+    fluent-plugin-jq (0.2.0)
       fluentd (>= 0.14.10, < 2)
+      multi_json (~> 1.13)
       ruby-jq (~> 0.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
     cool.io (1.5.3)
+    coveralls (0.8.21)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.14.1)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.4)
+      tins (~> 1.6)
     dig_rb (1.0.1)
+    docile (1.1.5)
     fluentd (1.1.0)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
@@ -22,7 +30,8 @@ GEM
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
-    msgpack (1.2.2)
+    json (2.1.0)
+    msgpack (1.2.4)
     multi_json (1.13.1)
     power_assert (1.1.1)
     rake (12.3.0)
@@ -31,13 +40,22 @@ GEM
     serverengine (2.0.6)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
+    simplecov (0.14.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
     strptime (0.2.3)
+    term-ansicolor (1.6.0)
+      tins (~> 1.0)
     test-unit (3.2.7)
       power_assert
+    thor (0.19.4)
     thread_safe (0.3.6)
-    tzinfo (1.2.4)
+    tins (1.16.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2017.3)
+    tzinfo-data (1.2018.3)
       tzinfo (>= 1.0.0)
     yajl-ruby (1.3.1)
 
@@ -46,9 +64,10 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.14)
+  coveralls (~> 0.8)
   fluent-plugin-jq!
   rake (~> 12.0)
   test-unit (~> 3.0)
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

data/Makefile

@@ -1,6 +1,7 @@
-.PHONY: test
+SHELL=/bin/bash
 
 # this is supposed to be used as travis build script
+.PHONY: test
 test:
 	@for n in 3 4 5; do \
 	   docker run -it --rm -v $$(pwd):/app -w /app ruby:2.$$n-alpine /app/run_ci.sh; \

data/README.md

@@ -1,9 +1,18 @@
-# fluent-plugin-jq [![Build Status](https://travis-ci.org/Gimi/fluent-plugin-jq.svg?branch=master)](https://travis-ci.org/Gimi/fluent-plugin-jq)
+# fluent-plugin-jq
 
-[Fluentd](https://fluentd.org/) formatter plugin to format events with [jq](https://stedolan.github.io/jq/).
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-jq.svg)](https://badge.fury.io/rb/fluent-plugin-jq)
+[![Build Status](https://travis-ci.org/Gimi/fluent-plugin-jq.svg?branch=master)](https://travis-ci.org/Gimi/fluent-plugin-jq)
+
+A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains two plugins:
+* `jq` formatter - a formatter plugin formats inputs using jq filters.
+* `jq_transformer` - a filter plugin transform inputs.
 
 ## Installation
 
+See also: [Plugin Management](https://docs.fluentd.org/v1.0/articles/plugin-management).
+
+Before you install this plugin, please make sure that `libjq` has been installed on your machine. For example, it's called `jq-dev` on alpine linux; while on debian, and ubuntu, you will find it as the `libjq-dev` package.
+
 ### RubyGems
 
 ```
@@ -26,11 +35,45 @@ $ bundle
 
 ## Configuration
 
-### jq_program (string) (required)
+See also: [Output Plugin Overview](https://docs.fluentd.org/v1.0/articles/output-plugin-overview).
+
+### `jq` Formatter
+
+#### Example
+
+```
+<match **>
+  @type stdout
+  <format>
+    @type jq
+    jq '"\(.time | todate) [\(.logLevel | ascii_upcase)] \(.log)"'
+  </format>
+</match>
+```
+
+In the example above, it will format the input with the jq filter specified in the `jq` parameter before sendint it to stdout. For example, given an input like
+
+```javascript
+{"time": 1520030594, "logLevel": "info", "log": "this is an example."}
+```
+
+What will be printed in stdout is
+
+```
+2018-03-02T22:43:14Z [INFO] this is an example.
+```
+
+#### Parameters
 
-The jq program used to format income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used.
+##### @type (string) (required)
 
-### on_error (enum) (optional)
+This must be `jq`.
+
+##### jq (string) (required)
+
+The jq filter for formatting income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used.
+
+##### on_error (enum) (optional)
 
 Defines the behavior on error happens when formatting an event. "skip" will skip the event; "ignore" will ignore the error and return the JSON representation of the original event; "raise_error" will raise a RuntimeError.
 
@@ -38,24 +81,49 @@ Available values: skip, ignore, raise_error
 
 Default value: `ignore`.
 
-### Example
+### `jq_transformer` Filter
+
+#### Example
+
+```
+<filter **>
+  @type jq_transformer
+  jq .record + {time, tag}
+</filter>
+```
+
+The above example will transform the input event (the record) by adding two more fields: `time` (the event time from fluentd) and `tag` (the event tag), to it. For example, given an input event like:
 
+```javascript
+{"logLevel": "info", "log": "this is an example."}
 ```
-<format>
-  @type jq
-  jq_program .message
-  on_error raise_error
-</format>
+
+with tag = `"some.tag"` and time = `1520030594`. Then the new event will like
+
+```javascript
+{"logLevel": "info", "log": "this is an example.", "time": 1520030594, "tag": "some.tag"}
 ```
 
-You can run
+#### Parameters
+
+##### @type (string) (required)
+
+This must be `jq_transformer`.
+
+##### jq (string) (required)
+
+The jq filter used to transform the input. The result of the filter should return an object. If after applying the transforming the new event is not an object (a hash), the event will be dropped.
+
+### Built-in Example
+
+Once you clone the project from github, you can run the following commands to see a real example for the plugins.
 
 ```
 $ rake build_example
 $ rake run_example
 ```
 
-to build an example docker image, and run it to see how it works. You can run `$ rake rm_example` to delete the image afterward.
+The above commands will build the example to a docker image, and run it. You can run `$ rake rm_example` to delete the image afterward.
 
 ## Copyright
 

data/example/fluent.conf

@@ -1,13 +1,18 @@
 <source>
   @type dummy
   tag dummy
-  dummy {"message": "some message", "some_other_key": "this is irrelevant."}
+  dummy {"log": "everything goes well", "stream": "stdout"}
 </source>
 
+<filter dummy>
+  @type jq_transformer
+  jq .record + {tag, time}
+</filter>
+
 <match dummy>
   @type stdout
   <format>
     @type jq
-    jq_program .message
+    jq '"\(.time | todate) [\(.tag)] \(.stream) => \(.log)"'
   </format>
 </match>

data/fluent-plugin-jq.gemspec

@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jq"
-  spec.version = "0.1.0"
+  spec.version = "0.2.0"
   spec.authors = ["Zhimin (Gimi) Liang"]
   spec.email   = ["liang.gimi@gmail.com"]
 
-  spec.summary       = %q{Fluentd formatter plugin uses the jq engine.}
-  spec.description   = %q{A Fluentd formatter plugin allows you to format events using jq filters.}
+  spec.summary       = %q{Fluentd plugins uses the jq engine.}
+  spec.description   = %q{fluent-plungin-jq is a collection of fluentd plugins which uses the jq engine to transform or format fluentd events.}
   spec.homepage      = "https://github.com/Gimi/fluent-plugin-jq"
   spec.license       = "Apache-2.0"
 
@@ -21,6 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "coveralls", "~> 0.8"
+
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
   spec.add_runtime_dependency "ruby-jq", "~> 0.1"
+  spec.add_runtime_dependency "multi_json", "~> 1.13"
 end

data/lib/fluent/plugin/filter_jq_transformer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/filter"
+
+module Fluent
+  module Plugin
+    class JqTransformerFilter < Fluent::Plugin::Filter
+      Fluent::Plugin.register_filter("jq_transformer", self)
+
+      desc 'The jq filter used to transform the input. The result of the filter should return an object.'
+      config_param :jq, :string
+
+      def initialize
+	super
+	require "jq"
+      end
+
+      def configure(conf)
+	super
+	JQ::Core.new @jq
+      rescue JQ::Error
+	raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
+      end
+
+      def filter(tag, time, record)
+	new_record = JQ(MultiJson.dump(tag: tag, time: time, record: record)).search(@jq).first
+	return new_record if new_record.is_a?(Hash)
+
+	log.error "jq filter #{@jq} did not return a hash, skip this record."
+	nil
+      rescue JQ::Error
+	log.error "Failed to transform #{MultiJson.dump record} with #{@jq}, error: #{$!.message}"
+	nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_jq.rb

@@ -1,4 +1,5 @@
-#
+# frozen_string_literal: true
+
 # Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,12 +19,13 @@ require "fluent/plugin/formatter"
 module Fluent
   module Plugin
     class JqFormatter < Fluent::Plugin::Formatter
-      EMPTY_STRING = ''.freeze
-
       Fluent::Plugin.register_formatter("jq", self)
 
       desc 'The jq program used to format income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used.'
-      config_param :jq_program
+      config_param :jq, :string, default: nil
+
+      desc 'The jq program used to format income events. The result of the program should only return one item of any kind (a string, an array, an object, etc.). If it returns multiple items, only the first will be used. DEPRECATED.'
+      config_param :jq_program, :string, deprecated: 'use jq instead.', default: nil
 
       desc 'Defines the behavior on error happens when formatting an event. "skip" will skip the event; "ignore" will ignore the error and return the JSON representation of the original event; "raise_error" will raise a RuntimeError.'
       config_param :on_error, :enum, list: [:skip, :ignore, :raise_error], default: :ignore
@@ -35,21 +37,25 @@ module Fluent
 
       def configure(conf)
 	super
-	JQ::Core.new @jq_program
+
+	@jq = @jq_program unless @jq
+	raise Fluent::ConfigError, "jq is required." unless @jq
+
+	JQ::Core.new @jq
       rescue JQ::Error
-	raise Fluent::ConfigError, "Could not parse jq_program, error: #{$!.message}"
+	raise Fluent::ConfigError, "Could not parse jq filter #{@jq}, error: #{$!.message}"
       end
 
       def format(tag, time, record)
-	item = JQ(record.to_json).search(@jq_program).first
+	item = JQ(MultiJson.dump(record)).search(@jq).first
 	return item if item.instance_of?(String)
-	item.to_json
+	MultiJson.dump item
       rescue JQ::Error
-	msg = "Failed to format #{record.to_json} with #{@jq_program}, error: #{$!.message}"
+	msg = "Failed to format #{record.to_json} with #{@jq}, error: #{$!.message}"
 	log.error msg
 	case @on_error
 	when :skip
-	  return EMPTY_STRING
+	  return ''
 	when :ignore
 	  return record.to_json
 	when :raise_error

data/test/helper.rb

@@ -1,8 +1,10 @@
 $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 require "test-unit"
 require "fluent/test"
-require "fluent/test/driver/formatter"
 require "fluent/test/helpers"
+require 'coveralls'
+
+Coveralls.wear!
 
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
 Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_filter_jq_transformer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "helper"
+require "fluent/test/driver/filter"
+require "fluent/plugin/filter_jq_transformer"
+
+class JqTransformerFilterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "it should require jq" do
+    assert_raise(Fluent::ConfigError) { create_driver '' }
+  end
+
+  test "it should raise error on invalid jq program" do
+    e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
+    assert_match(/compile error/, e.message)
+  end
+
+  test "it should work on tag" do
+    d = create_driver 'jq "{tag}"'
+    record = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", event_time, record)
+    assert_equal record["tag"], "some.tag"
+  end
+
+  test "it should work on time" do
+    d = create_driver 'jq "{time: .time | gmtime }"'
+    now = event_time
+    record = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", now, record)
+    assert_equal record["time"][0..5],
+      Time.at(now).utc.to_a[0..5].reverse.tap { |a| a[1] = a[1] - 1 }
+  end
+
+  test "it should work on record" do
+    d = create_driver 'jq "{message: .record.log, stream: .record.source}"'
+    record = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", event_time, record)
+    assert_equal record["message"], "this is a log"
+    assert_equal record["stream"], "stdout"
+  end
+
+  test "it should skip if it does not return a hash" do
+    record = {"log" => "this is a log", "source" => "stdout"}
+
+    d = create_driver 'jq ".tag"'
+    assert_nil d.instance.filter("some.tag", event_time, record)
+
+    d = create_driver 'jq ".time"'
+    assert_nil d.instance.filter("some.tag", event_time, record)
+
+    d = create_driver 'jq "[.time]"'
+    assert_nil d.instance.filter("some.tag", event_time, record)
+  end
+
+  test "it should only return one object" do
+    d = create_driver 'jq ".record, {tag, time}"'
+    event = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", event_time, event)
+    assert_equal event, record
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::JqTransformerFilter).configure(conf)
+  end
+end

data/test/plugin/test_formatter_jq.rb

@@ -1,43 +1,44 @@
 require "helper"
-require "fluent/plugin/formatter_jq.rb"
+require "fluent/test/driver/formatter"
+require "fluent/plugin/formatter_jq"
 
 class JqFormatterTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
   end
 
-  test "it should require jq_program" do
+  test "it should require jq parameter" do
     assert_raise(Fluent::ConfigError) { create_driver '' }
   end
 
-  test "it should raise error on invalid jq program" do
-    e = assert_raise(Fluent::ConfigError) { create_driver 'jq_program blah' }
-    assert_match /compile error/, e.message
+  test "it should raise error on invalid jq parameter" do
+    e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
+    assert_match(/compile error/, e.message)
   end
 
   test "it should work" do
-    d = create_driver 'jq_program .log'
+    d = create_driver 'jq .log'
     record = {"log" => "this is a log", "source" => "stdout"}
     text = d.instance.format("some.tag", event_time, record)
     assert_equal record["log"], text
   end
 
   test "it should ignore error by default" do
-    d = create_driver 'jq_program .[1]'
+    d = create_driver 'jq .[1]'
     record = {"log" => "some message"}
     text = d.instance.format "some.tag", event_time, record
     assert_equal record.to_json, text
   end
 
   test "it can skip on error" do
-    d = create_driver "jq_program .[1]\non_error skip"
+    d = create_driver "jq .[1]\non_error skip"
     record = {"log" => "some message"}
     text = d.instance.format "some.tag", event_time, record
     assert_equal '', text
   end
 
   test "it can raise error on error" do
-    d = create_driver "jq_program .[1]\non_error raise_error"
+    d = create_driver "jq .[1]\non_error raise_error"
     record = {"log" => "some message"}
     assert_raise(RuntimeError) {
       d.instance.format "some.tag", event_time, record

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jq
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Zhimin (Gimi) Liang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-11 00:00:00.000000000 Z
+date: 2018-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -86,13 +100,29 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-description: A Fluentd formatter plugin allows you to format events using jq filters.
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+description: fluent-plungin-jq is a collection of fluentd plugins which uses the jq
+  engine to transform or format fluentd events.
 email:
 - liang.gimi@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGES.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -102,9 +132,11 @@ files:
 - example/Dockerfile
 - example/fluent.conf
 - fluent-plugin-jq.gemspec
+- lib/fluent/plugin/filter_jq_transformer.rb
 - lib/fluent/plugin/formatter_jq.rb
 - run_ci.sh
 - test/helper.rb
+- test/plugin/test_filter_jq_transformer.rb
 - test/plugin/test_formatter_jq.rb
 homepage: https://github.com/Gimi/fluent-plugin-jq
 licenses:
@@ -126,10 +158,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.5
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: Fluentd formatter plugin uses the jq engine.
+summary: Fluentd plugins uses the jq engine.
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_jq_transformer.rb
 - test/plugin/test_formatter_jq.rb