fastly_fluent 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 908f50379a7eebc384dadea58ff75cab3452ee55
+  data.tar.gz: 4bff227b7727af33b38d45afef1600b0b1c02935
+SHA512:
+  metadata.gz: 1c4125fdf6248b2a62f5fd1916c85749e4ac3921c689f62555f3d7913c5a98cdd65f16df999f499a84dd03e74988494231be2ff75302d10ff8b2e6d7428d74ed
+  data.tar.gz: 3b33e3ed78c7bcea9c13b69ee51eea4ef764c6d4c5bce813e7af94da3bde49a7ae1488cd772c4a619ba87c763ae51ae4cd5d9e3b47220b8c86c5f533c6557011

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fastly_fluent.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Benjamin Bryant
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# FastlyTd
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fastly_td'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fastly_td
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fastly_fluent.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\
+
+Gem::Specification.new do |spec|
+  spec.name          = "fastly_fluent"
+  spec.version       = '0.0.1'
+  spec.authors       = ["Benjamin Bryant"]
+  spec.email         = ["benjaminhbryant@gmail.com"]
+  spec.description   = %q{fluent plugin for JSON encoded fastly syslogs}
+  spec.summary       = %q{}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_fastly.rb

@@ -0,0 +1,124 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  class FastlyInput < Input
+    Plugin.register_input('fastly', self)
+
+    #FASTLY_REGEXP = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]+)\s(?<fastly_host>[^ ]+)\s(?<tag>[^\[]\[\d+\]\:)\s+(?<message>.*)$/
+    FASTLY_REGEXP = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]+)\s(?<fastly_host>[^ ]+)\s(?<tag>[^\[]+)\[\d+\]:\s(?<message>.*)$/
+    FASTLY_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+    #{"method="}%t {"code"}%>s {"request"}%r
+
+    def initialize
+      super
+      require 'cool.io'
+      require 'fluent/plugin/socket_util'
+    end
+
+    config_param :port, :integer, :default => 5140
+    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :tag, :string
+    config_param :protocol_type, :default => :udp do |val|
+      case val.downcase
+      when 'tcp'
+        :tcp
+      when 'udp'
+        :udp
+      else
+        raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
+      end
+    end
+
+    def configure(conf)
+      super
+      @parser  = TextParser::RegexpParser.new(FASTLY_REGEXP, {'time_format' => FASTLY_TIME_FORMAT})
+    end
+
+    def start
+      callback = method(:receive_data)
+
+      @loop = Coolio::Loop.new
+      @handler = listen(callback)
+      @loop.attach(@handler)
+
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @handler.close
+      @thread.join
+    end
+
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    protected
+
+    def receive_data(data, addr)
+
+      @parser.call(data) { |time, record|
+        unless time && record
+          log.warn "invalid syslog message", :data => data
+          return
+        end
+
+        pri = record.delete('pri').to_i
+
+        tag = record.delete('tag')
+
+        message = JSON.parse(record.delete('message'))
+        message.each do |k,v|
+          record[k] = v unless v == "(null)"
+        end
+
+
+        emit(tag, time, record)
+      }
+    rescue => e
+      log.error data.dump, :error => e.to_s
+      log.error_backtrace
+    end
+
+    private
+
+    def listen(callback)
+      log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      if @protocol_type == :udp
+        @usock = SocketUtil.create_udp_socket(@bind)
+        @usock.bind(@bind, @port)
+        SocketUtil::UdpHandler.new(@usock, log, 2048, callback)
+      else
+        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback)
+      end
+    end
+
+    def emit(tag, time, record)
+      Engine.emit(tag, time, record)
+    rescue => e
+      log.error "fastly failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,31 @@
+require 'test/unit'
+require 'fileutils'
+require 'fluent/log'
+require 'fluent/test'
+
+require 'fluent/plugin/in_fastly'
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
+
+$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)

data/test/plugin/test_fastly_fluent.rb

@@ -0,0 +1,153 @@
+require 'fluent/test'
+require 'helper'
+
+class FastlyInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/socket_util'
+  end
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind 127.0.0.1
+    tag fastly
+  ]
+
+  IPv6_CONFIG = %[
+    port #{PORT}
+    bind ::1
+    tag fastly
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::FastlyInput).configure(conf)
+  end
+
+  def test_configure
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+      assert_equal PORT, d.instance.port
+      assert_equal k, d.instance.bind
+    }
+  end
+
+  def test_time_format
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+
+      tests = [
+        {'msg' => "<134>2014-07-10T23:18:15Z cache-hk91 td.server.requests[11226]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\"}", 'expected' => Time.new(2014,7,10,23,18,15).to_i },
+
+        {'msg' => "<134>2014-07-13T14:57:46Z cache-dfw1829 td.server.requests[524]: {\"ip\":\"166.137.213.198\",\"client_id\":\"17a8e5f8f64a9a7c85d1ccab51bcfdf6\",\"status\":200,\"user_agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\",\"null_val\":\"(null)\"}", 'expected' => Time.new(2014,07,13,14,57,46).to_i}
+
+      ]
+
+
+      d.run do
+        u = Fluent::SocketUtil.create_udp_socket(k)
+        u.connect(k, PORT)
+        tests.each {|test|
+          u.send(test['msg'], 0)
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+
+      tests.each_index  {|i|
+
+        assert_equal(tests[i]['expected'], emits[i][1])
+      }
+    }
+  end
+
+  def test_msg_size
+    d = create_driver
+    tests = create_test_case
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_msg_size_with_tcp
+    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    tests = create_test_case
+
+    d.run do
+      tests.each {|test|
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          s.send(test['msg'], 0)
+        end
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_msg_size_with_same_tcp_connection
+    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    tests = create_test_case
+
+    d.run do
+      TCPSocket.open('127.0.0.1', PORT) do |s|
+        tests.each {|test|
+          s.send(test['msg'], 0)
+        }
+      end
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_msg_size_with_json_format
+    d = create_driver([CONFIG, 'format json'].join("\n"))
+    time = Time.parse('2013-09-18 12:00:00 +0900').to_i
+    tests = ['Hello!', 'Syslog!'].map { |msg|
+      event = {'time' => time, 'message' => msg}
+      {'msg' => '<6>' + event.to_json + "\n", 'expected' => msg}
+    }
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def create_test_case
+    # actual syslog message has "\n"
+    [
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+    ]
+  end
+
+  def compare_test_result(emits, tests)
+    emits.each_index { |i|
+      assert_equal('fastly.kern.info', emits[0][0]) # <6> means kern.info
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+    }
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: fastly_fluent
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Benjamin Bryant
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: fluent plugin for JSON encoded fastly syslogs
+email:
+- benjaminhbryant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fastly_fluent.gemspec
+- lib/fluent/plugin/in_fastly.rb
+- test/helper.rb
+- test/plugin/test_fastly_fluent.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: ''
+test_files:
+- test/helper.rb
+- test/plugin/test_fastly_fluent.rb