fluentd-tcp-capturer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7ffa5c092d3847b323da8e1e6a06f07673266c6d
+  data.tar.gz: 6d98befca1fd50bd1ac59b8ed5aad4c508b30675
+SHA512:
+  metadata.gz: 727f0e018fc9dd4ac3d251a9cb5fec1ccd4b97a9329ec8214cfbb08f9e2d458b625b025072c38a55018828506b96b3ed631f7c999c0b5d574c9111deda753caf
+  data.tar.gz: 0386bd0aafe7da24a6e34729acf5e3366f6962900ece230865b7c59c713333279396cda34802bc91fc212d88bd59d989b6173bff00b60b8e4f7ec727f4f4e57f

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in capturing-message-fluentd.gemspec
+gemspec

data/README.md

@@ -0,0 +1,70 @@
+# Fluentd TCP capturer
+
+`fluentd-tcp-capturer` is a tool to inspect/dump/handle message to Fluentd TCP input, to:
+
+- debug a message to fluentd from somewhere
+- try other configuration on other fluentd node
+
+without changing Fluentd configuration.
+
+## Installation
+
+```shell
+$ gem install 'fluentd-tcp-capturer'
+```
+
+Then command `fm-cap` becomes available.
+
+
+## Usage
+
+```shell
+Usage: fm-cap [options]
+    -d, --device DEVICE              Device name [default: eth0]
+    -p, --port PORT                  Fluentd port to capture [default: 24224]
+        --forward-host HOST          If set, message will be forwarded to other Fluentd host
+        --forward-port PORT          Fluentd port to forward message (used when --forward-host is set)
+        --debug                      Set loglevel DEBUG
+```
+
+### Dump mode
+
+This mode captures tcp packet to Fluentd, dump it in the terminal.
+
+```shell
+# TODO
+$ sudo fm-cap
+I, [2017-03-03T22:41:31.141436 #14088]  INFO -- : Start capturing lo0/port=24224
+2017-03-03 13:41:34 +0000 | tag=test.20170303224134 msg={"name"=>"John", "age"=>15}
+2017-03-03 13:41:46 +0000 | tag=test.20170303224145 msg={"name"=>"Michel", "age"=>16}
+```
+
+You can specify other network device, also port number of Fluentd.
+
+```shell
+$ sudo fm-cap -d lo0
+$ sudo fm-cap -p 4567
+```
+
+### Transfer mode
+
+This mode captures tcp packet, transfer it to other Fluentd tcp input.
+
+```shell
+$ sudo fm-cap --forward-host other-fluentd-node --forward-port 4567
+I, [2017-03-03T22:46:31.878876 #14564]  INFO -- : Start capturing lo0/port=24224
+I, [2017-03-03T22:46:34.577661 #14564]  INFO -- : Forwarded message to other-fluentd-node:4567
+I, [2017-03-03T22:46:41.460288 #14564]  INFO -- : Forwarded message to other-fluentd-node:4567
+I, [2017-03-03T22:46:42.461110 #14564]  INFO -- : Forwarded message to other-fluentd-node:4567
+```
+
+## TODO
+
+- Support timezone in the dumpped message.
+- Dump message over embed Fluend.
+- Support other protocol, e.g. UDP
+- Tests ...
+
+## Patch
+
+Welcome

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/fm-cap

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+require "optparse"
+require "capture"
+require "logger"
+
+options = {
+  device: "eth0",
+  port: 24224,
+  forward_host: nil,
+  forward_port: 0,
+  output_file_path: nil
+}
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+
+
+OptionParser.new do |opts|
+  opts.on("-d DEVICE", "--device DEVICE", "Device name [default: eth0]") do |v|
+    options[:device] = v
+  end
+  opts.on("-p PORT", "--port PORT", "Fluentd port to capture [default: 24224]") do |v|
+    options[:port] = v.to_i
+  end
+  opts.on("--forward-host HOST", "If set, message will be forwarded to other Fluentd host") do |v|
+    options[:forward_host] = v
+  end
+  opts.on("--forward-port PORT", "Fluentd port to forward message (used when --forward-host is set)") do |v|
+    options[:forward_port] = v.to_i
+  end
+  opts.on("--debug", "Set loglevel DEBUG") do |v|
+    logger.level = Logger::DEBUG
+  end
+  # TODO
+  # opts.on("-o VALUE", "--output VALUE", "Output file path to dump message from Fluentd (used when --forward-host is set)") do |v|
+  #   options[:output_file_path] = v
+  # end
+  # TODO max_bytes, timeout
+  opts.parse!(ARGV)
+end
+
+
+exec = Capturing::Exec.new(device: options[:device], port: options[:port], logger: logger)
+runner = if options[:forward_host].nil?
+  Capturing::PrintRunner.new(exec, writer: STDOUT)
+else
+  # if options[:output_file_path].nil?
+  #   STDERR.write "-o/--output must be set"
+  # end
+  Capturing::ForwardRunner.new(exec, host: options[:forward_host], port: options[:forward_port],
+                               output: options[:output_file_path])
+end
+
+
+exec.logger.info "Start capturing #{options[:device]}/port=#{options[:port]}"
+begin
+  runner.run!
+rescue Interrupt
+  runner.destroy!
+  exit 0
+rescue => e
+  exec.logger.error e
+  exit 1
+end

data/fluentd-tcp-capturer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluentd-tcp-capturer"
+  spec.version       = "0.1.0"
+  spec.authors       = ["takumakanari"]
+  spec.email         = ["chemtrails.t@gmail.com"]
+
+  spec.summary       = %q{Fluentd message capturer}
+  spec.description   = %q{A tool to inspect/dump/handle message to Fluentd TCP input.}
+  spec.homepage      = "https://github.com/takumakanari/fluentd-tcp-capturer"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "ruby-pcap"
+  spec.add_runtime_dependency "threadpool"
+  spec.add_runtime_dependency "fluentd", [">= 0.12", "< 2"]
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/capture.rb

@@ -0,0 +1,160 @@
+require "pcap"
+require "logger"
+
+module Capturing
+
+  class Runner
+
+    def initialize(exec)
+      @exec = exec
+      @pcaplet = Pcap::Capture.open_live(exec.device, exec.max_bytes, true, exec.timeout)
+      @pcaplet.setfilter(Pcap::Filter.new("port #{exec.port} and tcp", @pcaplet))
+    end
+
+    def run!
+      @pcaplet.each_packet do |packet|
+        if packet.tcp? && packet.tcp_data_len > 0
+          @exec.logger.debug "Handle message packets: #{packet}"
+          begin
+            on_packet packet.tcp_data
+          rescue => e
+            @exec.logger.error e
+          end
+        end
+      end
+    end
+
+    def on_packet(data)
+      raise NotImplementedError.new "on_packet(packet)"
+    end
+
+    def destroy!
+      @pcaplet.close
+    end
+  end
+
+  class PrintRunner < Runner
+    require "fluent/engine"
+    require "fluent/time"
+
+    def initialize(e, writer:)
+      super e
+      @writer = writer
+      @unpacker = Fluent::Engine.msgpack_factory.unpacker
+      @time_formatter = Fluent::TimeFormatter.new("%Y-%m-%d %H:%M:%S %z", false, nil) # TODO support format
+    end
+
+    def on_packet(data)
+      @unpacker.feed_each(data) do |msg|
+        tag, entries = msg
+        entries.each do |e|
+          @writer.write(format_message(tag, e))
+        end
+      end
+    end
+
+    def format_message(tag, entry)
+      time, record = entry
+      "#{@time_formatter.format(time)} | tag=#{tag} msg=#{record.inspect}\n"
+    end
+  end
+
+  class ForwardRunner < Runner
+    require "socket"
+    require "threadpool"
+
+    def initialize(e, host:, port:, output:)
+      super e
+      @host = host
+      @port = port
+      @output = output
+      @use_embed_fluentd = false # TODO support this mode
+
+      start_embed_fluentd if @use_embed_fluentd
+      @sock = new_socket_to_forward
+      @messaging_thread_pool = ThreadPool.new(4) # TODO configurable
+    end
+
+    def on_packet(data)
+      @messaging_thread_pool.process{ @sock.write data }
+      @exec.logger.info "Forwarded message to #{@host}:#{@port}"
+    end
+
+    private
+    def new_socket_to_forward
+      max_retry = 10
+      begin
+        TCPSocket.new(@host, @port)
+      rescue Errno::ECONNREFUSED
+        raise if max_retry == 0
+        sleep 1
+        max_retry -= 1
+        retry
+      end
+    end
+
+    def start_embed_fluentd
+      Thread.start {
+        Dir.mktmpdir do |dir|
+          conf = File.join(dir, "fluent.conf")
+          output_file = File.join(dir, 'output')
+          @exec.logger.info "Output message to '#{output_file}'" # TODO set file path
+          File.write(conf, <<-EOF)
+            <source>
+              @type forward
+              port #{@port}
+            </source>
+            <match **>
+              @type file
+              path #{@output}
+              buffer_type memory
+              append true
+              flush_interval 0s
+            </match>
+          EOF
+          FluentdEmbed.new(conf).boot
+        end
+      }
+    end
+  end
+
+  class Exec
+    attr_reader :device, :max_bytes, :timeout, :port
+
+    DEFAULT_LOGGER ||= Logger.new(STDOUT)
+
+    def initialize(device: "eth0", max_bytes: 1460, timeout: 1000, port: 24224, logger: nil)
+      @device = device
+      @max_bytes = max_bytes
+      @timeout = timeout
+      @port = port
+      @logger = logger
+    end
+
+    def logger
+      @logger.nil? ? DEFAULT_LOGGER : @logger
+    end
+  end
+
+  class FluentdEmbed
+    require "fluent/version"
+    require "fluent/supervisor"
+
+    def initialize(config_path)
+      @opts = {
+        config_path: config_path,
+        plugin_dirs: [],
+        log_level: 2,
+        libs: [],
+        suppress_repeated_stacktrace: true,
+        use_v1_config: true,
+        supervise: true,
+        standalone_worker: true
+      }
+    end
+
+    def boot
+      Fluent::Supervisor.new(@opts).run_worker
+    end
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: fluentd-tcp-capturer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- takumakanari
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-pcap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: threadpool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: A tool to inspect/dump/handle message to Fluentd TCP input.
+email:
+- chemtrails.t@gmail.com
+executables:
+- fm-cap
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/fm-cap
+- fluentd-tcp-capturer.gemspec
+- lib/capture.rb
+homepage: https://github.com/takumakanari/fluentd-tcp-capturer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Fluentd message capturer
+test_files: []