test-jq 0.5.1

data/Rakefile

@@ -0,0 +1,25 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+task :build_example do
+  sh 'docker build -t fluent-plugin-jq-example -f example/Dockerfile .'
+end
+
+task :rm_example do
+  sh 'docker image rm fluent-plugin-jq-example'
+end
+
+task :run_example do
+  sh 'docker run -it --rm fluent-plugin-jq-example'
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/VERSION

@@ -0,0 +1 @@
+0.5.1

data/example/Dockerfile

@@ -0,0 +1,19 @@
+FROM fluent/fluentd:v1.1.0
+LABEL maintainer="Zhimin (Gimi) Liang (https://github.com/Gimi)"
+
+COPY . /tmp/fluent-plugin-jq/
+
+RUN apk update \
+ && apk add --no-cache jq \
+ && apk add --no-cache --virtual .build-deps \
+        build-base \
+        git \
+	ruby-dev \
+ && cd /tmp/fluent-plugin-jq \
+ && gem build fluent-plugin-jq.gemspec \
+ && gem install *.gem \
+ && apk del .build-deps \
+ && rm -rf /var/cache/apk/* \
+ && cp /tmp/fluent-plugin-jq/example/fluent.conf /fluentd/etc/ \
+ && cp /tmp/fluent-plugin-jq/example/statsd.sample / \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

data/example/fluent.conf

@@ -0,0 +1,42 @@
+<source>
+  @type dummy
+  tag raw.dummy
+  dummy {"log": "everything goes well", "stream": "stdout"}
+</source>
+
+<source>
+  @type tail
+  tag stats
+  path /statsd.sample
+  read_from_head true
+  <parse>
+    @type jq
+    jq 'split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = ($pair[1][:-2] | tonumber))'
+  </parse>
+</source>
+
+<match raw.dummy>
+  @type jq
+  jq .record | to_entries
+  remove_tag_prefix raw
+</match>
+
+<filter dummy>
+  @type jq_transformer
+  jq .record + {tag, time}
+</filter>
+
+<match dummy>
+  @type stdout
+  <format>
+    @type jq
+    jq '"\(.time | todate) [\(.tag)] \(.key) => \(.value)"'
+  </format>
+</match>
+
+<match stats>
+  @type stdout
+  <format>
+    @type json
+  </format>
+</match>

data/example/statsd.sample

@@ -0,0 +1,4 @@
+cpu.usage:10|g,cpu.free:90|g
+memory.usage:100|g,memory.rss:80|g
+network.rx:12.34|g,network.tx:56.78|g
+login.success:100|c,login.failed:20|c

data/fluent-plugin-jq.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "test-jq"
+  spec.version = File.read('VERSION')
+  spec.authors       = ['Rock']
+  spec.email         = ['rbaek@splunk.com']
+
+  spec.summary       = %q{Fluentd plugins uses the jq engine.}
+  spec.description   = %q{fluent-plungin-jq is a collection of fluentd plugins which uses the jq engine to transform or format fluentd events.}
+  spec.homepage      = "https://github.com/rockb1017/fluent-plugin-jq"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = Dir.glob('*').select { |f| not (File.directory?(f) || f.start_with?('.')) } +
+    Dir.glob('lib/**/**.rb') +
+    Dir.glob('example/**/**')
+  spec.test_files    = Dir.glob('test/**/**.rb')
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "coveralls", "~> 0.8"
+
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+  spec.add_runtime_dependency "multi_json", "~> 1.13"
+end

data/lib/fluent/plugin/filter_jq_transformer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/filter"
+require "fluent/plugin/jq_mixin"
+
+module Fluent
+  module Plugin
+    class JqTransformerFilter < Fluent::Plugin::Filter
+      Fluent::Plugin.register_filter("jq_transformer", self)
+
+      include JqMixin
+
+      config_set_desc :jq, 'The jq filter used to transform the input. The result of the filter should return an object.'
+
+      def filter(tag, time, record)
+	new_record = jq_transform tag: tag, time: time, record: record
+	return new_record if new_record.is_a?(Hash)
+
+	log.error "jq filter #{@jq} did not return a hash, skip this record."
+	nil
+      rescue JqError
+	log.error "Filter failed with #{@jq}#{log.on_debug { ' on ' + MultiJson.dump(tag: tag, time: time, record: record) }}, error: #{$!.message}"
+	nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_jq.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/formatter"
+require "fluent/plugin/jq_mixin"
+
+module Fluent
+  module Plugin
+    class JqFormatter < Fluent::Plugin::Formatter
+      Fluent::Plugin.register_formatter("jq", self)
+
+      include JqMixin
+
+      config_set_desc :jq, 'The jq filter used to format income events. If the result returned from the filter is not a string, it will be encoded as a JSON string.'
+
+      desc 'Defines the behavior on error happens when formatting an event. "skip" will skip the event; "ignore" will ignore the error and return the JSON representation of the original event; "raise_error" will raise a RuntimeError.'
+      config_param :on_error, :enum, list: [:skip, :ignore, :raise_error], default: :ignore
+
+      def initialize
+	super
+      end
+
+      def format(tag, time, record)
+	item = jq_transform record
+	if item.instance_of?(String)
+	  item
+	else
+	  MultiJson.dump item
+	end
+      rescue JqError
+	msg = "Format failed with #{@jq}#{log.on_debug { ' on ' + MultiJson.dump(record) }}, error: #{$!.message}"
+	log.error msg
+	case @on_error
+	when :skip
+	  return ''
+	when :ignore
+	  return MultiJson.dump(record)
+	when :raise_error
+	  raise msg
+	end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/jq_mixin.rb

@@ -0,0 +1,50 @@
+require 'shellwords'
+require 'multi_json'
+
+module JqMixin
+  JqError = Class.new(RuntimeError)
+
+  def self.included(plugin)
+    plugin.config_param :jq, :string
+  end
+
+  def configure(conf)
+    super
+    p = start_process(null_input: true)
+    err = p.read
+    raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{err}" if err =~ /compile error/m
+  rescue
+    raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
+  ensure
+    p.close if p # if `super` fails, `p` will be `nil`
+  end
+
+  def start
+    super
+    @jq_process = start_process
+    @lock = Thread::Mutex.new
+  end
+
+  def shutdown
+    @jq_process.close rescue nil
+    super
+  end
+
+  def start_process(filter: @jq, null_input: false)
+    IO.popen(%Q"jq #{'-n' if null_input} --unbuffered -c '#{filter}' 2>&1", 'r+')
+  end
+
+  def jq_transform(object)
+    result = @lock.synchronize do
+      @jq_process.puts MultiJson.dump(object)
+      @jq_process.gets
+    end
+    MultiJson.load result
+  rescue MultiJson::ParseError
+    raise JqError.new(result)
+  rescue Errno::EPIPE
+    @jq_process.close
+    @jq_process = start_process
+    retry
+  end
+end

data/lib/fluent/plugin/out_jq.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/output'
+require 'fluent/plugin/jq_mixin'
+
+module Fluent::Plugin
+  class JqOutput < Output
+    Fluent::Plugin.register_output('jq', self)
+    helpers :event_emitter
+
+    include JqMixin
+
+    config_set_desc :jq, 'The jq filter used to transform the input. If the filter returns an array, each object in the array will be a new record.'
+
+    desc 'The prefix to be removed from the input tag when outputting a new record.'
+    config_param :remove_tag_prefix, :string, default: ''
+
+    def multi_workers_ready?
+      true
+    end
+
+    def process(tag, es)
+      new_es = Fluent::MultiEventStream.new
+      es.each do |time, record|
+	begin
+	  new_records = jq_transform tag: tag, time: time, record: record
+	  new_records = [new_records] unless new_records.is_a?(Array)
+	  new_records.each { |new_record| new_es.add time, new_record }
+	rescue JqError
+	  log.error "Process failed with #{@jq}#{log.on_debug {' on ' + MultiJson.dump(record)}}, error: #{$!.message}"
+	end
+      end
+
+      new_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+      router.emit_stream(new_tag, new_es)
+    end
+  end
+end

data/lib/fluent/plugin/parser_jq.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/parser"
+require 'fluent/plugin/jq_mixin'
+
+module Fluent
+  module Plugin
+    class JqParser < Fluent::Plugin::Parser
+      Fluent::Plugin.register_parser("jq", self)
+
+      include JqMixin
+
+      config_set_desc :jq, 'The jq filter used to format the input. The result of the filter must return an object.'
+
+      def parse(text)
+	record = jq_transform text
+	if record.is_a?(Hash)
+	  yield parse_time(record), record
+	else
+	  log.error "jq filter #{@jq} did not return a hash, skip this record."
+	end
+      rescue JqError
+	log.error "Parse failed with #{@jq}#{log.on_debug {' on ' + text}}, error: #{$!.message}"
+	nil
+      end
+    end
+  end
+end

data/run_ci.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+apk add --no-cache build-base jq-dev \
+  && bundle \
+  && rake

data/test/helper.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/helpers"
+require 'coveralls'
+
+Coveralls.wear!
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_filter_jq_transformer.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "helper"
+require "fluent/test/driver/filter"
+require "fluent/plugin/filter_jq_transformer"
+
+class JqTransformerFilterTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  teardown do
+    @driver.instance.shutdown if @driver
+  end
+
+  test "it should require jq" do
+    assert_raise(Fluent::ConfigError) { create_driver '' }
+  end
+
+  test "it should raise error on invalid jq program" do
+    e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
+    assert_match(/compile error/, e.message)
+  end
+
+  test "it should work on tag" do
+    d = create_driver 'jq "{tag}"'
+    record = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", event_time, record)
+    assert_equal record["tag"], "some.tag"
+  end
+
+  test "it should work on time" do
+    d = create_driver 'jq "{time: .time | gmtime }"'
+    now = event_time
+    record = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", now, record)
+    assert_equal record["time"][0..5],
+      Time.at(now).utc.to_a[0..5].reverse.tap { |a| a[1] = a[1] - 1 }
+  end
+
+  test "it should work on record" do
+    d = create_driver 'jq "{message: .record.log, stream: .record.source}"'
+    record = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", event_time, record)
+    assert_equal record["message"], "this is a log"
+    assert_equal record["stream"], "stdout"
+  end
+
+  test "it should skip if it does not return a hash" do
+    record = {"log" => "this is a log", "source" => "stdout"}
+
+    d = create_driver 'jq ".tag"'
+    assert_nil d.instance.filter("some.tag", event_time, record)
+
+    d = create_driver 'jq ".time"'
+    assert_nil d.instance.filter("some.tag", event_time, record)
+
+    d = create_driver 'jq "[.time]"'
+    assert_nil d.instance.filter("some.tag", event_time, record)
+  end
+
+  test "it should only return one object" do
+    d = create_driver 'jq ".record, {tag, time}"'
+    event = {"log" => "this is a log", "source" => "stdout"}
+    record = d.instance.filter("some.tag", event_time, event)
+    assert_equal event, record
+  end
+
+  private
+
+  def create_driver(conf)
+    @driver = Fluent::Test::Driver::Filter.new(Fluent::Plugin::JqTransformerFilter).configure(conf).tap { |d| d.instance.start }
+  end
+end