fluent-plugin-pcapng 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-pcapng.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 enukane
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# Fluent::Plugin::Pcapng
+
+fluent-plugin-pcapng is an input plug-in for Fluentd.
+It runs tshark with specified configuration and extract given packet fields.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-pcapng'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-pcapng
+
+## Usage
+
+Add the following lines into your fluentd config.
+
+simple case:
+
+```
+<source>
+  type pcapng
+
+  interface eth0
+  fields frame.time,eth.dst,eth.src,eth.type
+</source>
+```
+
+advanced case:
+
+```
+<source>
+  type pcapng
+
+  tag mypcap
+  interface eth0
+  fields frame.time,frame.time_epoch,ip.src,ip.dst,ip.proto
+  types time,double,string,string,long
+  convertdot __
+</source>
+```
+
+## Configuration
+
+|name|type|required?|default|description|
+|:---|:---|:--------|:------|:----------|
+| interface | string | required | "eth0" | interface to capture |
+| fields | array | required | none | list of field to extract (-e on tshark) |
+| types | array | optional | "string" for all | list of type for each field ("long", "double", "string", "time") |
+| convertdot | string | optional | none | convert "." in field name (for outputing int DB who doesn't accept "dot" in schema) |
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fluent-plugin-pcapng.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-pcapng"
+  spec.version       = "0.0.1"
+  spec.authors       = ["enukane"]
+  spec.email         = ["enukane@glenda9.org"]
+  spec.description   = %q{Fluentd plugin for tshark (pcapng) monitoring from specified interface}
+  spec.summary       = %q{Fluentd input plugin for monitoring packets received in specified interface}
+  spec.homepage      = "https://github.com/enukane/fluent-plugin-pcapng"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/fluent/plugin/in_pcapng.rb

@@ -0,0 +1,165 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class PcapngInput < Input
+    Plugin.register_input('pcapng', self)
+
+    require 'open3'
+    require 'csv'
+    require 'time'
+
+    LONG="long"
+    DOUBLE="double"
+    STRING="string"
+    TIME="time"
+
+    def initialize
+      super
+    end
+
+    config_param :tag, :string, :default => "pcapng"
+    config_param :interface, :string, :default => 'ge0'
+    config_param :convertdot, :string, :default => nil
+    config_param :fields, :default => [] do |val|
+      val.split(',')
+    end
+    config_param :types, :default => [] do |val|
+      val.split(',')
+    end
+
+    def configure(conf)
+      super
+
+      if @fields != nil and @fields == []
+        raise ConfigError, "'fields' option is required on pcapng input"
+      end
+
+      if @types and @types.length != @fields.length
+        raise ConfigError, "'types' length must be equal to 'fields' length"
+      end
+    end
+
+    def start
+      super
+
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      if @th_tshark and @th_tshark.alive?
+        Process.kill("INT", @th_tshark.pid)
+      end
+      @thread.join
+    rescue => e
+      log.error "pcapng failed to shutdown", :error => e.to_s,
+        :error_class => e.class.to_s
+      log.error_backtrace e.backtrace
+    end
+
+    def run
+      options = build_options(@fields)
+      cmdline = "tshark -i #{@interface} -T fields -E separator=\",\" -E quote=d #{options}"
+      print cmdline + "\n"
+      stdin, stdout, stderr, @th_tshark = *Open3.popen3(cmdline)
+
+      while @th_tshark.alive?
+        collect_tshark_output(stdout)
+      end
+    rescue => e
+      log.error "unexpected error", :error => e.to_s
+      log.error_backtrace e.backtrace
+    end
+
+    def build_options(fields)
+      options = ""
+      fields.each do |field|
+        options += "-e \"#{field}\" "
+      end
+      return options
+    end
+
+    def collect_tshark_output(stdout)
+      collected = []
+      begin
+        readlines_nonblock(stdout).each do |line|
+          array = CSV.parse(line).flatten
+          collected << array
+        end
+      rescue => e
+        log.error "pcapng failed to read or parse line", :error => e.to_s,
+          :error_class => e.class.to_s
+      end
+
+      collected.each do |ary|
+        router.emit(@tag, Engine.now, generate_record(ary))
+      end
+    rescue => e
+      log.error "pcapng failed to collect output from tshark",
+        :error => e.to_s,
+        :error_class => e.class.to_s
+    end
+
+    def readlines_nonblock(io)
+      @nbbuffer = "" if @nbbuffer == nil
+      @nbbuffer += io.read_nonblock(65535)
+      lines = []
+      while idx = @nbbuffer.index("\n")
+        lines << @nbbuffer[0..idx-1]
+        @nbbuffer = @nbbuffer[idx+1..-1]
+      end
+      return lines
+    rescue
+      return []
+    end
+
+    def generate_record(array)
+      fields = @fields
+      if fields.length != array.length
+        return {}
+      end
+      carray = convert_types(array)
+      if @convertdot
+        fields = fields.map{|field| field.gsub(".", @convertdot)}
+      end
+      return Hash[[fields, carray].transpose]
+    end
+
+    def convert_types(array)
+      return [array, @types].transpose.map{|val, type|
+        convert_type(val, type)
+      }
+    end
+
+    def convert_type val, type
+      v = val.to_s.gsub("\"", "")
+      v = "" if val == nil
+      case type
+      when LONG
+        if v.is_a?(String) and v.match(/^0x[0-9a-fA-F]+$/)
+          v = v.hex
+        else
+          v = v.to_i
+        end
+      when DOUBLE
+        v = v.to_f
+      when TIME
+        v = Time.parse(v)
+      end
+      return v
+    end
+  end
+end

data/sample/pcapng.conf.sample

@@ -0,0 +1,15 @@
+<source>
+  type pcapng
+  id pcap_input
+
+  tag mypcap
+  interface wlan2
+  fields radiotap.channel.freq,wlan.fc.type_subtype,wlan.ta
+  types long,long,string
+  convertdot __
+</source>
+
+<match mypcap>
+  type stdout
+  id stdout_output
+</match>

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-pcapng
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- enukane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin for tshark (pcapng) monitoring from specified interface
+email:
+- enukane@glenda9.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-pcapng.gemspec
+- lib/fluent/plugin/in_pcapng.rb
+- sample/pcapng.conf.sample
+homepage: https://github.com/enukane/fluent-plugin-pcapng
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 573394323
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 573394323
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd input plugin for monitoring packets received in specified interface
+test_files: []