fluent-plugin-heroku-syslog 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c1ba40b8e6dea0a28ffed82b25787897c0ead301
+  data.tar.gz: e2e430813032e6445a4bf0344dca1f746580ca09
+SHA512:
+  metadata.gz: cc75128973dfcb608908f1f930e9cf4e6eb436457a12d1cbfddc1308c902e8e56d77df21f34b153c9f32c3c9a2c3b3c47e6490eb31df67cb1c3c699d35dfbf53
+  data.tar.gz: 4743b87ccc0fec77eb82613a49308700995aa305a133ed68c55c39631339b4b356e14c6b1dbabe0475491ee69dddbfea839cd21cd2404aa8f4a44cae51f5f98e

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+test.conf

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-heroku-syslog.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2014- Kazuyuki Honda
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,33 @@
+# fluent-plugin-heroku-syslog
+
+fluent plugin to drain heroku syslog.
+
+[![Build Status](https://travis-ci.org/hakobera/fluent-plugin-heroku-syslog.png?branch=master)](https://travis-ci.org/hakobera/fluent-plugin-heroku-syslog)
+
+## Component
+
+### HerokuSyslogInput
+
+Plugin to accept syslog input from [heroku syslog drains](https://devcenter.heroku.com/articles/logging#syslog-drains).
+
+## Configuration
+
+```
+<source>
+  type heroku_syslog
+  port 5140
+  bind 0.0.0.0
+  tag  heroku
+</source>
+```
+
+## TODO
+
+- Implement authentication logic or filter like HTTP basic auth.
+
+## Copyright
+
+- Copyright
+  - Copyright(C) 2014- Kazuyuki Honda (hakobera)
+- License
+  - Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-heroku-syslog.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-heroku-syslog"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Kazuyuki Honda"]
+  gem.email         = ["hakobera@gmail.com"]
+  gem.description   = %q{fluent plugin to drain heroku syslog}
+  gem.summary       = %q{fluent plugin to drain heroku syslog}
+  gem.homepage      = "https://github.com/hakobera/fluent-plugin-heroku-syslog"
+  gem.license       = "APLv2"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_development_dependency "rake"
+end

data/lib/fluent/plugin/in_heroku_syslog.rb

@@ -0,0 +1,221 @@
+module Fluent
+  class HerokuSyslogInput < Input
+    Plugin.register_input('heroku_syslog', self)
+
+    OCTET_COUNTING_REGEXP = /^([0-9]+)\s+(.*)/
+    SYSLOG_REGEXP = /^\<([0-9]+)\>[0-9]*(.*)/
+    SYSLOG_ALL_REGEXP = /^\<(?<pri>[0-9]+)\>[0-9]* (?<time>[^ ]*) (?<drain_id>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*) (?<pid>[a-zA-Z0-9\.]+)? *(?<message>.*)$/
+    TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+
+    FACILITY_MAP = {
+      0   => 'kern',
+      1   => 'user',
+      2   => 'mail',
+      3   => 'daemon',
+      4   => 'auth',
+      5   => 'syslog',
+      6   => 'lpr',
+      7   => 'news',
+      8   => 'uucp',
+      9   => 'cron',
+      10  => 'authpriv',
+      11  => 'ftp',
+      12  => 'ntp',
+      13  => 'audit',
+      14  => 'alert',
+      15  => 'at',
+      16  => 'local0',
+      17  => 'local1',
+      18  => 'local2',
+      19  => 'local3',
+      20  => 'local4',
+      21  => 'local5',
+      22  => 'local6',
+      23  => 'local7'
+    }
+
+    PRIORITY_MAP = {
+      0  => 'emerg',
+      1  => 'alert',
+      2  => 'crit',
+      3  => 'err',
+      4  => 'warn',
+      5  => 'notice',
+      6  => 'info',
+      7  => 'debug'
+    }
+
+    def initialize
+      super
+      require 'cool.io'
+      require 'fluent/plugin/socket_util'
+    end
+
+    config_param :port, :integer, :default => 5140
+    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :tag, :string
+
+    def configure(conf)
+      super
+
+      parser = TextParser.new
+      if parser.configure(conf, false)
+        @parser = parser
+      else
+        @parser = nil
+        @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
+      end
+    end
+
+    def start
+      if @parser
+        callback = method(:receive_data_parser)
+      else
+        callback = method(:receive_data)
+      end
+
+      @loop = Coolio::Loop.new
+      @handler = listen(callback)
+      @loop.attach(@handler)
+
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @handler.close
+      @thread.join
+    end
+
+    def run
+      @loop.run
+    rescue
+      $log.error "unexpected error", :error=>$!.to_s
+      $log.error_backtrace
+    end
+
+    protected
+    def receive_data_parser(data)
+      m = SYSLOG_REGEXP.match(data)
+      unless m
+        $log.debug "invalid syslog message: #{data.dump}"
+        return
+      end
+      pri = m[1].to_i
+      text = m[2]
+
+      time, record = @parser.parse(text)
+      unless time && record
+        return
+      end
+
+      emit(pri, time, record)
+
+    rescue
+      $log.warn data.dump, :error=>$!.to_s
+      $log.debug_backtrace
+    end
+
+    def receive_data(data)
+      m = SYSLOG_ALL_REGEXP.match(data)
+      unless m
+        $log.debug "invalid syslog message", :data=>data
+        return
+      end
+
+      pri = nil
+      time = nil
+      record = {}
+
+      m.names.each {|name|
+        if value = m[name]
+          case name
+          when "pri"
+            pri = value.to_i
+          when "time"
+            time = @time_parser.parse(value.gsub(/ +/, ' ').gsub(/\.[0-9]+/, ''))
+          else
+            record[name] = value
+          end
+        end
+      }
+
+      time ||= Engine.now
+
+      emit(pri, time, record)
+
+    rescue
+      $log.warn data.dump, :error=>$!.to_s
+      $log.debug_backtrace
+    end
+
+    private
+
+    def listen(callback)
+      $log.debug "listening heroku syslog socket on #{@bind}:#{@port}"
+      Coolio::TCPServer.new(@bind, @port, TcpHandler, callback)
+    end
+
+    def emit(pri, time, record)
+      facility = FACILITY_MAP[pri >> 3]
+      priority = PRIORITY_MAP[pri & 0b111]
+
+      tag = "#{@tag}.#{facility}.#{priority}"
+
+      Engine.emit(tag, time, record)
+    rescue => e
+      $log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+    end
+
+    class TcpHandler < Coolio::Socket
+      def initialize(io, on_message)
+        super(io)
+        if io.is_a?(TCPSocket)
+          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        end
+        $log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+        @on_message = on_message
+        @buffer = "".force_encoding('ASCII-8BIT')
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        @buffer << data
+        pos = 0
+
+        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+        while i = @buffer.index("\n", pos)
+          msg = @buffer[pos..i]
+
+          # Support Octet Counting 
+          # https://tools.ietf.org/html/rfc6587#section-3.4.1
+          m = OCTET_COUNTING_REGEXP.match(msg)
+          valid = true
+          if m
+            msg_len = m[1].to_i - 1
+            msg = m[2]
+
+            if msg_len != msg.length
+              $log.debug "invalid syslog message length", :expected => msg_len, :actual => msg.length, :data => msg
+              valid = false
+            end
+          end
+          @on_message.call(msg) if valid
+          pos = i + 1
+        end
+        @buffer.slice!(0, pos) if pos > 0
+      rescue => e
+        $log.error "syslog error", :error => e, :error_class => e.class
+        close
+      end
+
+      def on_close
+        $log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fileutils'
+require 'fluent/log'
+require 'fluent/test'
+
+$log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_DEBUG)
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
+
+require 'fluent/plugin/in_heroku_syslog'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_heroku_syslog.rb

@@ -0,0 +1,127 @@
+require 'helper'
+
+class HerokuSyslogInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind 127.0.0.1
+    tag heroku.syslog
+  ]
+
+  IPv6_CONFIG = %[
+    port #{PORT}
+    bind ::1
+    tag syslog
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::HerokuSyslogInput).configure(conf)
+  end
+
+  def test_configure
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+      assert_equal PORT, d.instance.port
+      assert_equal k, d.instance.bind
+    }
+  end
+
+  def test_time_format
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+
+      tests = [
+        {
+            'msg' => "92 <13>1 2014-01-29T06:25:52.589365+00:00 d.916a3e50-efa1-4754-aded-ffffffffffff app web.1 foo\n",
+            'expected' => Time.strptime('2014-01-29T06:25:52+00:00', '%Y-%m-%dT%H:%M:%S%z').to_i
+        },
+        {
+            'msg' => "92 <13>1 2014-01-30T07:35:00.123456+09:00 d.916a3e50-efa1-4754-aded-ffffffffffff app web.1 bar\n",
+            'expected' => Time.strptime('2014-01-30T07:35:00+09:00', '%Y-%m-%dT%H:%M:%S%z').to_i
+        }
+      ]
+
+      d.run do
+        tests.each {|test|
+          TCPSocket.open(k, PORT) do |s|
+            s.send(test['msg'], 0)
+          end
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      $log.debug emits
+      emits.each_index {|i|
+        $log.debug emits[i][1]
+        assert_equal(tests[i]['expected'], emits[i][1])
+      }
+    }
+  end
+
+  def test_msg_size
+    d = create_driver
+    tests = create_test_case
+
+    d.run do
+      tests.each {|test|
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          s.send(test['msg'], 0)
+        end
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_msg_size_with_same_tcp_connection
+    d = create_driver
+    tests = create_test_case
+
+    d.run do
+      TCPSocket.open('127.0.0.1', PORT) do |s|
+        tests.each {|test|
+          s.send(test['msg'], 0)
+        }
+      end
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def create_test_case
+    # actual syslog message has "\n"
+    msgs = [
+      {'msg' => '<13>1 2014-01-01T01:23:45.123456+00:00 d.916a3e50-efa1-4754-aded-ffffffffffff app web.1 ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+      {'msg' => '<13>1 2014-01-01T01:23:45.123456+00:00 d.916a3e50-efa1-4754-aded-ffffffffffff app web.1 ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+    ]
+
+    msgs.each do |msg|
+      msg['msg'] = "#{msg['msg'].length} #{msg['msg']}"
+    end
+
+    msgs
+  end
+
+  def compare_test_result(emits, tests)
+    emits.each_index {|i|
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert_equal('d.916a3e50-efa1-4754-aded-ffffffffffff', emits[i][2]['drain_id'])
+      assert_equal('app', emits[i][2]['ident'])
+      assert_equal('web.1', emits[i][2]['pid'])
+    }
+  end
+
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-heroku-syslog
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kazuyuki Honda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluent plugin to drain heroku syslog
+email:
+- hakobera@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-heroku-syslog.gemspec
+- lib/fluent/plugin/in_heroku_syslog.rb
+- test/helper.rb
+- test/plugin/test_in_heroku_syslog.rb
+homepage: https://github.com/hakobera/fluent-plugin-heroku-syslog
+licenses:
+- APLv2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: fluent plugin to drain heroku syslog
+test_files:
+- test/helper.rb
+- test/plugin/test_in_heroku_syslog.rb