fluent-plugin-stdin 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b2ea3c32514a386aa25828404b061916931bc597
+  data.tar.gz: ae4d073b0fcd6ad01b785a3633f16ec547809d11
+SHA512:
+  metadata.gz: e9b0869419b7c5d07220cd15c8a0e286f3d22f313baad513e76cf912eb25ee79d9cef05e67c4085aedf5fdbb6ae643ed702f8bfc11fd8e734ba8aaf2be7135bc
+  data.tar.gz: 406870d59ffbc5cb42590cf846f73268f2f32b222212ab369dad03b213418dca1e0a69045d535e24b0a306a99b325a3270891bf4b66f8034494457b4b8ff4663

data/.travis.yml

@@ -0,0 +1,20 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - 2.2
+  - ruby-head
+  - rbx
+
+branches:
+  only:
+    - master
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,41 @@
+# Fluentd plugin for reading events from stdin
+
+Reading events from stdin for testing or small bulk loading.
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-stdin
+
+## Configuration
+
+    <source>
+      type stdin
+
+      # Input pattern. It depends on Parser plugin
+      format none
+
+      # Optional. default is stdin.events
+      tag test.input
+    </source>
+
+After that, you can send logs to fluentd via stdin like below.
+
+    cat /path/to/logs | fluentd -c stdin.conf
+
+This plugin works on only non-daemon mode.
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td>Masahiro Nakagawa <repeatedly@gmail.com></td>
+  </tr>
+  <tr>
+    <td>Copyright</td><td>Copyright (c) 2015- Masahiro Nakagawa</td>
+  </tr>
+  <tr>
+    <td>License</td><td>MIT License</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,13 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-stdin.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-stdin"
+  gem.description = "Fluentd plugin for reading events from stdin"
+  gem.homepage    = "https://github.com/repeatedly/fluent-plugin-stdin"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Masahiro Nakagawa"]
+  gem.email       = "repeatedly@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", ">= 0.10.17"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", "~> 3.0.2"
+end

data/lib/fluent/plugin/in_stdin.rb

@@ -0,0 +1,78 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class StdinInput < Input
+    Plugin.register_input('stdin', self)
+
+    config_param :format, :string
+    config_param :delimiter, :string, :default => "\n"
+    config_param :tag, :string, :default => 'stdin.events'
+
+    def configure(conf)
+      super
+
+      @parser = Plugin.new_parser(@format)
+      @parser.configure(conf)
+    end
+
+    def start
+      @buffer = "".force_encoding('ASCII-8BIT')
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @thread.join
+    end
+
+    def run
+      while true
+        begin
+          @buffer << $stdin.sysread(4000)
+          pos = 0
+
+          while i = @buffer.index(@delimiter, pos)
+            msg = @buffer[pos...i]
+            emit_event(msg)
+            pos = i + @delimiter.length
+          end
+          @buffer.slice!(0, pos) if pos > 0
+        rescue IOError, EOFError => e
+          # ignore above exceptions because can't re-open stdin automatically
+          break
+        rescue => e
+          log.error "unexpected error", :error=> e.to_s
+          log.error_backtrace
+          break
+        end
+      end
+    end
+
+    def emit_event(msg)
+      @parser.parse(msg) { |time, record|
+        unless time && record
+          log.warn "pattern not match: #{msg.inspect}"
+          return
+        end
+
+        router.emit(@tag, time, record)
+      }
+    rescue => e
+      log.error msg.dump, :error => e, :error_class => e.class
+      log.error_backtrace
+    end
+  end
+end

data/test/test_in_stdin.rb

@@ -0,0 +1,58 @@
+require 'fluent/test'
+require 'fluent/plugin/in_stdin'
+
+class StdinInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    r, w = IO.pipe
+    $stdin = r
+    @writer = w
+  end
+
+  def teardown
+    $stdin = STDIN
+  end
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::StdinInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver("format none")
+    assert_equal 'stdin.events', d.instance.tag
+    assert_equal "\n", d.instance.delimiter
+  end
+
+  {
+    'none' => [
+      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
+      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
+    ],
+    'json' => [
+      {'msg' => {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n", 'expected' => 'tcptest1'},
+      {'msg' => {'k' => 'tcptest2', 'message' => 456}.to_json + "\n", 'expected' => 456},
+    ]
+  }.each { |format, test_cases|
+    define_method("test_msg_size_#{format}") do
+      d = create_driver("format #{format}")
+      tests = test_cases
+
+      d.run do
+        tests.each { |test|
+          @writer.write test['msg']
+        }
+        @writer.close
+        sleep 1
+      end
+
+      compare_test_result(d.emits, tests)
+    end
+  }
+
+  def compare_test_result(emits, tests)
+    assert_equal(2, emits.size)
+    emits.each_index {|i|
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+    }
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-stdin
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Masahiro Nakagawa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+description: Fluentd plugin for reading events from stdin
+email: repeatedly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-stdin.gemspec
+- lib/fluent/plugin/in_stdin.rb
+- test/test_in_stdin.rb
+homepage: https://github.com/repeatedly/fluent-plugin-stdin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin for reading events from stdin
+test_files:
+- test/test_in_stdin.rb