fluent-plugin-sflow 0.1.0 → 0.2.0

data/fluent-plugin-sflow.gemspec

@@ -4,39 +4,25 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-sflow"
-  spec.version       = "0.1.0"
-  spec.authors       = ["enukane"]
-  spec.email         = ["enukane@glenda9.org"]
+  spec.version       = "0.2.0"
+  spec.authors       = ["Shintaro Kojima"]
+  spec.email         = ["goodies@codeout.net"]
 
-  spec.summary       = %q{sFlow plugin for Fluentd}
-  spec.description   = %q{sFlow input plugin for Fluentd}
-  spec.homepage      = "https://github.com/enukane/fluent-plugin-sflow"
-  spec.license       = "MIT"
+  spec.summary       = "sFlow plugin for Fluentd"
+  spec.description   = "sFlow v2 / v4 / v5 input plugin for Fluentd supporting many packet formats"
+  spec.homepage      = "https://github.com/codeout/fluent-plugin-sflow"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  gem_dir = File.expand_path(File.dirname(__FILE__)) + "/"
-  `git submodule --quiet foreach pwd`.split($\).each do |submodule_path|
-    Dir.chdir(submodule_path) do
-      submodule_relative_path = submodule_path.sub gem_dir, ""
-      # issue git ls-files in submodule's directory and
-      # prepend the submodule path to create absolute file paths
-      `git ls-files`.split($\).each do |filename|
-        spec.files << "#{submodule_relative_path}/#{filename}"
-      end
-    end
-  end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/sflowtool/extconf.rb"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_dependency "fluentd", "~> 0.12.40"
+  spec.add_development_dependency "bundler", "~> 1.12"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "test-unit"
-
-  spec.add_dependency "fluentd", "~> 0.14.10"
-  spec.add_dependency "bindata", "1.8.1"
-  spec.add_dependency "eventmachine", "~> 1.2.3"
+  spec.add_development_dependency "rake-compiler", "~> 1.0"
+  spec.add_development_dependency "test-unit",  "~> 3.0"
 end

data/lib/fluent/plugin/in_sflow.rb

@@ -1,45 +1,88 @@
-require 'fluent/plugin/input'
+require 'cool.io'
+require 'fluent/plugin/parser_sflow'
 
-require 'bindata'
-require 'eventmachine'
-require 'yaml'
 
-dir = 'sflow/lib/sflow'
-['models/ipv4header', 'models/tcpheader', 'models/udpheader', 'models/protocol', 'models/binary_models','parsers/parsers'].each do |req|
-  require File.join(dir, req)
-end
-
-#$:.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'sflow', 'lib'))
-#require 'sflow'
-
-module Fluent::Plugin
+module Fluent
   class SflowInput < Input
-    Fluent::Plugin.register_input("sflow", self)
-
-    helpers :server
+    Plugin.register_input('sflow', self)
 
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 6343
     config_param :tag, :string
+    config_param :max_bytes, :integer, default: 2048
+
 
     def configure(conf)
       super
-
-      # dummy data
-      $switch_hash = {}
+      @parser = Fluent::TextParser::SflowParser.new
     end
 
     def start
       super
+      @loop = Coolio::Loop.new
+      @handler = listen(method(:receive))
+      @loop.attach @handler
 
-      server_create(:in_sflow_server, @port, bind: @bind, proto: :udp, max_bytes: 2048) do |data, sock|
-        sflow = SflowParser.parse_packet(data)
-        router.emit(@tag, Fluent::EventTime.now, sflow)
-      end
+      @thread = Thread.new(&method(:run))
     end
 
     def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @handler.close
+      @thread.join
       super
     end
+
+    def run
+      @loop.run
+    rescue
+      log.error 'unexpected error', error_class: $!.class, error: $!.message
+      log.error_backtrace
+    end
+
+
+    protected
+
+    def receive(raw, exporter)
+      log.on_debug do
+        log.debug 'received sflow datagram', raw: raw, exporter: exporter
+      end
+
+      @parser.parse(raw, exporter) do |time, record|
+        if !time || !record
+          log.warn 'Failed to parse', raw: raw, exporter: exporter
+          return
+        end
+
+        router.emit(@tag, time, record)
+      end
+    rescue
+      log.warn 'Unexpected error on parsing',
+               raw: raw, exporter: exporter, error_class: $!.class, error: $!.message
+    end
+
+
+    private
+
+    def listen(callback)
+      log.info "listening sflow socket on #{@bind}:#{@port}"
+      @sock = SocketUtil.create_udp_socket(@bind)
+      @sock.bind @bind, @port
+      UdpHandler.new @sock, callback
+    end
+
+    class UdpHandler < Coolio::IO
+      def initialize(io, callback)
+        super io
+        @io = io
+        @callback = callback
+      end
+
+      def on_readable
+        msg, addr = @io.recvfrom_nonblock(4096)
+        @callback.call msg, addr[3]
+      end
+    end
   end
 end

data/lib/fluent/plugin/parser_sflow.rb

@@ -0,0 +1,24 @@
+require 'fluent/parser'
+require 'json'
+require 'sflowtool'
+
+
+module Fluent
+  class TextParser
+    class SflowParser < Parser
+      Plugin.register_parser('sflow', self)
+
+
+      def parse(raw, remote_host)
+        data = JSON.load(Sflowtool.parse(raw, remote_host))
+
+        # NOTE: sFlow datagram doesn't have timestamp field, but sysUpTime only
+        time = data['header']['unix_seconds_utc']
+
+        data['samples'].each do |sample|
+          yield time, data['header'].merge(sample)
+        end
+      end
+    end
+  end
+end

data/lib/sflowtool.rb

@@ -0,0 +1,17 @@
+require 'sflowtool/sflowtool'
+
+module Sflowtool
+  class << self
+    def parse(data, exporter)
+      purge_trailing_commas(receive_sflow_datagram(data, exporter))
+    end
+
+    private
+
+    def purge_trailing_commas(str)
+      str.gsub! ',]', ']'
+      str.gsub! ',}', '}'
+      str
+    end
+  end
+end

metadata

@@ -1,161 +1,112 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sflow
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
-- enukane
+- Shintaro Kojima
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-24 00:00:00.000000000 Z
+date: 2017-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
-  type: :development
+        version: 0.12.40
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 0.12.40
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: test-unit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.14.10
-  type: :runtime
+        version: '1.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.14.10
-- !ruby/object:Gem::Dependency
-  name: bindata
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.8.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.8.1
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: eventmachine
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.3
-  type: :runtime
+        version: '3.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.3
-description: sFlow input plugin for Fluentd
+        version: '3.0'
+description: sFlow v2 / v4 / v5 input plugin for Fluentd supporting many packet formats
 email:
-- enukane@glenda9.org
+- goodies@codeout.net
 executables: []
-extensions: []
+extensions:
+- ext/sflowtool/extconf.rb
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".gitmodules"
 - ".travis.yml"
 - Gemfile
-- LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
 - example/fluentd.conf
+- ext/sflowtool/extconf.rb
+- ext/sflowtool/fluent-plugin-sflow.c
+- ext/sflowtool/sflow.h
+- ext/sflowtool/sflow_v2v4.h
+- ext/sflowtool/sflowtool.c
 - fluent-plugin-sflow.gemspec
 - lib/fluent/plugin/in_sflow.rb
-- lib/sflow/Gemfile
-- lib/sflow/Gemfile.lock
-- lib/sflow/LICENSE.txt
-- lib/sflow/README.md
-- lib/sflow/Rakefile
-- lib/sflow/bin/bundler
-- lib/sflow/bin/rake
-- lib/sflow/bin/sflow.rb
-- lib/sflow/etc/config.yaml
-- lib/sflow/lib/sflow.rb
-- lib/sflow/lib/sflow/collector.rb
-- lib/sflow/lib/sflow/config.rb
-- lib/sflow/lib/sflow/models/binary_models.rb
-- lib/sflow/lib/sflow/models/ipv4header.rb
-- lib/sflow/lib/sflow/models/protocol.rb
-- lib/sflow/lib/sflow/models/tcpheader.rb
-- lib/sflow/lib/sflow/models/udpheader.rb
-- lib/sflow/lib/sflow/parsers/parsers.rb
-- lib/sflow/lib/sflow/snmp/iface_names.rb
-- lib/sflow/lib/sflow/storage/storage.rb
-- lib/sflow/lib/sflow/version.rb
-- lib/sflow/misc/kibana-schema.json
-- lib/sflow/misc/screen1.png
-- lib/sflow/sflow.gemspec
-- lib/sflow/test/lib/sflow/version_test.rb
-- lib/sflow/test/test_helper.rb
-homepage: https://github.com/enukane/fluent-plugin-sflow
-licenses:
-- MIT
+- lib/fluent/plugin/parser_sflow.rb
+- lib/sflowtool.rb
+homepage: https://github.com/codeout/fluent-plugin-sflow
+licenses: []
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -173,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: sFlow plugin for Fluentd

data/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "lib/sflow"]
-	path = lib/sflow
-	url = https://github.com/NETWAYS/sflow.git

data/LICENSE.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2017 enukane
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

data/lib/sflow/Gemfile

@@ -1,8 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in sflow.gemspec
-gemspec
-gem "bindata"
-gem "eventmachine"
-gem "json"
-gem "minitest"