fluent-plugin-input-jsonstream 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dc06d04f6fe9640b488591223d11a19e60f45fb6
+  data.tar.gz: 482a366b30a2733483820c944994b6812e72f498
+SHA512:
+  metadata.gz: 87bf0305b59d703578b6fb931d1b0299c57b8135c7f216600a96c959f467d7ed31b16fdbb28b9edaeccc47d7a55835620d38833566673bf804933835e0604b44
+  data.tar.gz: fdd242be75c8b5ee16de7eb827ed5d97a813ab10115914a002e4c640b428226fb189c9d0ede4b173ad8f97a776e51d6bfd0dec62d2c36fbd04e613cd595f0065

data/.gitignore

@@ -0,0 +1,11 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2.2
+  - 2.3.0
+  - 2.4.0
+
+gemfile:
+  - Gemfile
+
+branches:
+  only:
+    - master
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Daniel Malon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+# fluent-plugin-input-jsonstream
+
+[![Build Status](https://travis-ci.org/MerlinDMC/fluent-plugin-input-jsonstream.svg?branch=master)](https://travis-ci.org/MerlinDMC/fluent-plugin-input-jsonstream)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-input-jsonstream.svg)](http://badge.fury.io/rb/fluent-plugin-input-jsonstream)
+
+## Overview
+
+A streaming JSON input for [Fluentd](http://www.fluentd.org/).
+
+### Configuration
+
+Accept chunked JSON objects via TCP
+
+```
+<source>
+  @type jsonstream
+  tag example.jsonstream
+  bind 127.0.0.1
+  port 12345
+</source>
+
+<match example.jsonstream>
+  type stdout
+</match>
+```

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/fluent-plugin-input-jsonstream.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-input-jsonstream"
+  gem.description = "A streaming JSON input plugin for fluentd"
+  gem.license     = "MIT"
+  gem.homepage    = "https://github.com/MerlinDMC/fluent-plugin-input-jsonstream"
+  gem.summary     = gem.description
+  gem.version     = "0.1.0"
+  gem.authors     = ["Daniel Malon"]
+  gem.email       = "daniel.malon@me.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency 'yajl-ruby', '~> 1.0'
+
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
+end

data/lib/fluent/plugin/in_jsonstream.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+
+module Fluent::Plugin
+  class JSONStreamInput < Input
+    Fluent::Plugin.register_input('jsonstream', self)
+
+    helpers :server, :extract
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5170
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    config_param :blocking_timeout, :time, default: 0.5
+
+    def configure(conf)
+      super
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      server_create_connection(:in_jsonstream_server, @port, bind: @bind) do |conn|
+        parser = Yajl::Parser.new(:symbolize_keys => false)
+        parser.on_parse_complete = lambda { |record|
+          tag = extract_tag_from_record(record)
+          tag ||= @tag
+          time ||= extract_time_from_record(record) || Fluent::EventTime.now
+
+          # Use the recorded event time if available
+          time = record.delete('timestamp').to_i if record.key?('timestamp')
+
+          router.emit(tag, time, record)
+        }
+
+        conn.data do |data|
+          parser << data
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,57 @@
+require "test/unit"
+require "fluent/test"
+require "fluent/test/helpers"
+require "fluent/test/driver/input"
+
+include Fluent::Test::Helpers
+
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
+  case protocol
+  when :tcp
+    unused_port_tcp(num)
+  when :udp
+    unused_port_udp(num, bind: bind)
+  else
+    raise ArgumentError, "unknown protocol: #{protocol}"
+  end
+end
+
+def unused_port_tcp(num = 1)
+  ports = []
+  sockets = []
+  num.times do
+    s = TCPServer.open(0)
+    sockets << s
+    ports << s.addr[1]
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+PORT_RANGE_AVAILABLE = (1024...65535)
+
+def unused_port_udp(num = 1, bind: "0.0.0.0")
+  family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+  ports = []
+  sockets = []
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
+    u = UDPSocket.new(family)
+    if (u.bind(bind, port) rescue nil)
+      ports << port
+      sockets << u
+    else
+      u.close
+    end
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end

data/test/plugin/test_in_jsonstream.rb

@@ -0,0 +1,108 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require "fluent/plugin/in_jsonstream"
+
+class JSONStreamInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  BASE_CONFIG = %[
+    port #{PORT}
+    tag test
+  ]
+  CONFIG = BASE_CONFIG + %[
+    bind 127.0.0.1
+  ]
+
+  attr_reader :base_config
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::JSONStreamInput).configure(conf)
+  end
+
+  def create_tcp_socket(host, port, &block)
+    if block_given?
+      TCPSocket.open(host, port, &block)
+    else
+      TCPSocket.open(host, port)
+    end
+  end
+
+  def test_configure_requires_tag
+    assert_raise Fluent::ConfigError do
+      create_driver("")
+    end
+  end
+
+  def test_configuring_tag
+    d = create_driver(BASE_CONFIG)
+    assert_equal d.instance.tag, "test"
+  end
+
+  def test_simple_parse
+    payload = '{"foo":"bar"}'
+    expected = { 'foo' => 'bar' }
+
+    d = create_driver(CONFIG)
+    d.run(expect_records: 1) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        sock.send(payload, 0)
+      end
+    end
+
+    assert_equal 1, d.events.size
+    assert_equal "test", d.events[0][0]
+    assert d.events[0][1].is_a?(Fluent::EventTime)
+    assert_equal expected, d.events[0][2]
+  end
+
+  def test_chunked_parse
+    payloads = [
+      '{"fo', 'o": "b', 'ar"}'
+    ]
+    expected = { 'foo' => 'bar' }
+
+    d = create_driver(CONFIG)
+    d.run(expect_records: 1) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        payloads.each do |payload|
+          sock.send(payload, 0)
+        end
+      end
+    end
+
+    assert_equal 1, d.events.size
+    assert_equal "test", d.events[0][0]
+    assert d.events[0][1].is_a?(Fluent::EventTime)
+    assert_equal expected, d.events[0][2]
+  end
+
+  def test_chunked_multiple_parse
+    payloads = [
+      '{"fo', 'o": "b', 'ar"}',
+      '{"fo', 'o": "b', 'az"}'
+    ]
+    expecteds = [
+      { 'foo' => 'bar' },
+      { 'foo' => 'baz' }
+    ]
+
+    d = create_driver(CONFIG)
+    d.run(expect_records: 2) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        payloads.each do |payload|
+          sock.send(payload, 0)
+        end
+      end
+    end
+
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected, i|
+      assert_equal "test", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expecteds[i], d.events[i][2]
+    end
+  end
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-input-jsonstream
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Daniel Malon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A streaming JSON input plugin for fluentd
+email: daniel.malon@me.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-input-jsonstream.gemspec
+- lib/fluent/plugin/in_jsonstream.rb
+- test/helper.rb
+- test/plugin/test_in_jsonstream.rb
+homepage: https://github.com/MerlinDMC/fluent-plugin-input-jsonstream
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: A streaming JSON input plugin for fluentd
+test_files:
+- test/helper.rb
+- test/plugin/test_in_jsonstream.rb