fluent-plugin-jq 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85add62e095656dfaeb43bff27ae998d7ff39cd523a2f0d2d564d981a8e3ccf6
-  data.tar.gz: ec02443a917e479f171ecc44603f39b4f3c7f5e3ecbe629c0a2c4ce8eabe2bc4
+  metadata.gz: 6fef74bd93979266e240f96325d7686b004acb091e16264713e5ccb4431a632d
+  data.tar.gz: f7cc5b63c3863efd09a472a4d0f48c62e3879df6f9eb1ab095bee309f7dfe7e0
 SHA512:
-  metadata.gz: 4f340a9588d62e49758f7e8331768d912b4518876ef97945a82f9a6977fd516b23a32ece0712371f42bbd29607961e1ceb9ce2b0ce95daf8ba832d3a727ba247
-  data.tar.gz: 0601f3c89a55354fd844c6ef0152879d020a0af3c4d9980c9e5109622f25affd54ace34fe351f44e50eb825af137861721d90c015c74b19f97847c44b5341298
+  metadata.gz: 3e6335fd5b5099d5165182a28b889a7d7df007466f7c84ac1da31a0c709be0ff4a55baef6125b1351162a3174365140b22a47c110d42ba5263953693eced5179
+  data.tar.gz: df42a0cec7562e2a5882666923169a6871ea046c8c79a28ae9f45cd089f62f30498837948ca9801f162618c53daa5c6d741626642f6a858f3f9e1cdd0d6f555e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-jq (0.3.0)
+    fluent-plugin-jq (0.4.0)
       fluentd (>= 0.14.10, < 2)
       multi_json (~> 1.13)
       ruby-jq (~> 0.1)

data/README.md

@@ -3,10 +3,11 @@
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-jq.svg)](https://badge.fury.io/rb/fluent-plugin-jq)
 [![Build Status](https://travis-ci.org/Gimi/fluent-plugin-jq.svg?branch=master)](https://travis-ci.org/Gimi/fluent-plugin-jq)
 
-A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains three plugins:
+A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains four plugins:
 * `jq` formatter - a formatter plugin formats inputs using jq filters.
 * `jq_transformer` - a filter plugin transform inputs.
 * `jq` output - a output plugin uses jq filter to generate new events.
+* `jq` parser - a parser plugin uses jq filter to parse inputs.
 
 ## Installation
 
@@ -137,7 +138,7 @@ It generates two new events:
 
 ```javascript
 {"key": "logLevel", "value": "info"}
-{"key": log", "value": "this is an example."}
+{"key": "log", "value": "this is an example."}
 ```
 
 #### Parameters
@@ -157,6 +158,46 @@ Example: If `remove_tag_prefix` is set to 'foo', the input tag foo.bar.baz is tr
 
 Default value: `""`.
 
+### `jq` Parser
+
+#### Example
+
+```
+<source>
+  @type tail
+  tag tail.*
+  path /some/path/*
+  <parse>
+    @type jq
+    jq 'split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = ($pair[1][:-2] | tonumber))'
+  </parse>
+</source>
+```
+
+Given inputs like
+
+```
+cpu.usage:10|g,cpu.free:90|g
+memory.usage:100|g,memory.rss:80|g
+```
+
+It generates records:
+
+```javascript
+{"cpu.usage": 10, "cpu.free": 90}
+{"memory.usage": 100, "memory.rss": 80}
+```
+
+#### Parameters
+
+##### @type (string) (required)
+
+This must be `jq`.
+
+##### jq (string) (required)
+
+The jq filter used to parse inputs. The result of the filter must return an object, otherwise the result will be dropped.
+
 ### Built-in Example
 
 Once you clone the project from github, you can run the following commands to see a real example for the plugins.

data/example/Dockerfile

@@ -15,4 +15,5 @@ RUN apk update \
  && apk del .build-deps \
  && rm -rf /var/cache/apk/* \
  && cp /tmp/fluent-plugin-jq/example/fluent.conf /fluentd/etc/ \
+ && cp /tmp/fluent-plugin-jq/example/statsd.sample / \
  && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

data/example/fluent.conf

@@ -4,6 +4,17 @@
   dummy {"log": "everything goes well", "stream": "stdout"}
 </source>
 
+<source>
+  @type tail
+  tag stats
+  path /statsd.sample
+  read_from_head true
+  <parse>
+    @type jq
+    jq 'split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = ($pair[1][:-2] | tonumber))'
+  </parse>
+</source>
+
 <match raw.dummy>
   @type jq
   jq .record | to_entries
@@ -22,3 +33,10 @@
     jq '"\(.time | todate) [\(.tag)] \(.key) => \(.value)"'
   </format>
 </match>
+
+<match stats>
+  @type stdout
+  <format>
+    @type json
+  </format>
+</match>

data/example/statsd.sample

@@ -0,0 +1,4 @@
+cpu.usage:10|g,cpu.free:90|g
+memory.usage:100|g,memory.rss:80|g
+network.rx:12.34|g,network.tx:56.78|g
+login.success:100|c,login.failed:20|c

data/fluent-plugin-jq.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-jq"
-  spec.version = "0.3.0"
+  spec.version = "0.4.0"
   spec.authors = ["Zhimin (Gimi) Liang"]
   spec.email   = ["liang.gimi@gmail.com"]
 

data/lib/fluent/plugin/parser_jq.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/parser"
+
+module Fluent
+  module Plugin
+    class JqParser < Fluent::Plugin::Parser
+      Fluent::Plugin.register_parser("jq", self)
+
+      desc 'The jq filter used to format the input. The result of the filter must return an object.'
+      config_param :jq, :string
+
+      def initialize
+	super
+	require "jq"
+      end
+
+      def configure(conf)
+	super
+	@jq_filter = JQ::Core.new @jq
+      rescue JQ::Error
+	raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
+      end
+
+      def parse(text)
+	record = [].tap { |buf|
+	  @jq_filter.update(MultiJson.dump(text), false) { |r|
+	    buf << MultiJson.load("[#{r}]").first
+	  }
+	}.first
+	if record.is_a?(Hash)
+	  yield parse_time(record), record
+	else
+	  log.error "jq filter #{@jq} did not return a hash, skip this record."
+	end
+      rescue JQ::Error
+	log.error "Failed to parse #{text} with #{@jq}, error: #{$!.message}"
+	nil
+      end
+    end
+  end
+end

data/test/plugin/test_parser_jq.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "helper"
+require "fluent/test/driver/parser"
+require "fluent/plugin/parser_jq"
+
+class JqParserTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "it should require jq" do
+    assert_raise(Fluent::ConfigError) { create_driver '' }
+  end
+
+  test "it should raise error on invalid jq program" do
+    e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
+    assert_match(/compile error/, e.message)
+  end
+
+  test "it should work" do
+    d = create_driver 'jq split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = $pair[1])'
+    text = "name:jq,type:parser,author:Gimi"
+    expected_record = {"name" => "jq", "type" => "parser", "author" => "Gimi"}
+    d.instance.parse(text) { |time, record|
+      assert_equal expected_record, record
+    }
+    assert_nil d.instance.log.logs.find { |log| log =~ /\[error\]/ }
+  end
+
+  test "it should skip if it does not return a hash" do
+    d = create_driver 'jq split(",")'
+    text = "name:jq,type:parser,author:Gimi"
+    expected_record = {"name": "jq", "type": "parser", "author": "Gimi"}
+    d.instance.parse(text) { |time, record| assert false }
+    assert d.instance.log.logs.any? { |log| log =~ /\[error\]/ }
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::JqParser).configure(conf)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-jq
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Zhimin (Gimi) Liang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-23 00:00:00.000000000 Z
+date: 2018-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -131,15 +131,18 @@ files:
 - Rakefile
 - example/Dockerfile
 - example/fluent.conf
+- example/statsd.sample
 - fluent-plugin-jq.gemspec
 - lib/fluent/plugin/filter_jq_transformer.rb
 - lib/fluent/plugin/formatter_jq.rb
 - lib/fluent/plugin/out_jq.rb
+- lib/fluent/plugin/parser_jq.rb
 - run_ci.sh
 - test/helper.rb
 - test/plugin/test_filter_jq_transformer.rb
 - test/plugin/test_formatter_jq.rb
 - test/plugin/test_out_jq.rb
+- test/plugin/test_parser_jq.rb
 homepage: https://github.com/Gimi/fluent-plugin-jq
 licenses:
 - Apache-2.0
@@ -167,5 +170,6 @@ summary: Fluentd plugins uses the jq engine.
 test_files:
 - test/helper.rb
 - test/plugin/test_filter_jq_transformer.rb
+- test/plugin/test_parser_jq.rb
 - test/plugin/test_out_jq.rb
 - test/plugin/test_formatter_jq.rb