fluent-plugin-pull_forward 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9f729c6a8875eb13dfef8e811bf6681e62085b46
+  data.tar.gz: 8654a13d126608b371211e873b2741d61380484f
+SHA512:
+  metadata.gz: 427df78ec3facb73f152f1f710693c687cccbc6650759678f9d30b1312ca0ea34809cbb54f315b31c321f4dba4d16d590c893d3c9d06226b2cd1eebe5744d90b
+  data.tar.gz: 88fb138607c461e2e4f81854dd40548ef0b9f8d2c31f134443d7340588aca2d1c8aeed77fa1ed27053728abb70183c2c64ba4cc1f4138ddd84658fcf22cde9d7

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-pullforward.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,92 @@
+# fluent-plugin-pull_forward
+
+[Fluentd](http://fluentd.org) input/output plugin to forward data, by pulling/request-based transportation, over HTTPS.
+
+We can do with pull_forward:
+* transfer data into hosts in firewall by pulling
+* protect transferring route by HTTPS and basic authentication
+* fetch Fluentd events as JSON by HTTPS from any processes
+
+![plugin image](https://raw.githubusercontent.com/tagomoris/fluent-plugin-pull_forward/master/misc/plugin_image.png)
+
+## Configuration
+
+### PullForwardOutput
+
+Configure output plugin to transfer fluentd events to another fluentd nodes.
+
+```apache
+<match dummy>
+  type pull_forward
+  
+  buffer_path    /home/myname/tmp/fluentd_event.buffer
+  flush_interval 1m   ## default 1h
+  
+  self_hostname      ${hostname}
+  cert_auto_generate yes
+  # or
+  # "cert_file_path PATH", "private_key_path PATH" and "private_key_passphrase ..."
+  
+  <user>
+    username tagomoris
+    password foobar
+  </user>
+  <user>
+    username repeatedly
+    password booo
+  </user>
+</match>
+```
+
+PullForwardOutput uses PullPoolBuffer plugin. **DO NOT MODIFY buffer_type**. It uses buffer file, so `buffer_path` is required, and Not so short values are welcome for `flush_interval` because PullPoolBuffer make a file per flushes (and these are not removed until fetches of cluent/in\_pull\_forward).
+
+PullForward always requires SSL and basic authentication. SSL options and `<user>` sections are also required.
+
+### PullForwardInput
+
+Configure input plugin to fetch fluentd events from another fluentd nodes.
+
+```apache
+<source>
+  type pull_forward
+  
+  fetch_interval 10s
+  timeout 10s
+  
+  <server>
+    host host1.on.internet.example.com
+    username tagomoris
+    password foobar
+  </server>
+  <server>
+    host host2.on.internet.example.com
+    username tagomoris
+    password foobar
+  </server>
+</source>
+```
+
+PullForwardInput can fetch events from many nodes of `<server>`.
+
+### HTTPS fetch
+
+We can fluentd events from PullForwardOutput by normal HTTPS.
+
+```
+$ curl -k -s --user tagomoris:foobar https://localhost:24280/
+[
+  [ "test.foo", 1406915165, { "pos": 8, "hoge": 1 } ],
+  [ "test.foo", 1406915168, { "pos": 9, "hoge": 1 } ],
+  [ "test.foo", 1406915173, { "pos": 0, "hoge": 0 } ]
+]
+```
+
+## TODO
+
+* TESTS!
+
+## Copyright
+
+* Copyright (c) 2014- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/example/simple_in.conf

@@ -0,0 +1,19 @@
+<source>
+  type pull_forward
+  fetch_interval 10s
+  timeout 10s
+  <server>
+    host localhost
+    username moris
+    password foobar
+  </server>
+   <server>
+     host localhost
+     username moris
+     password fooba
+   </server>
+</source>
+
+<match test.**>
+  type stdout
+</match>

data/example/simple_out.conf

@@ -0,0 +1,15 @@
+<source>
+  type forward
+</source>
+
+<match test.**>
+  type pull_forward
+  buffer_path /tmp/pullforward.buffer.log
+  self_hostname ${hostname}
+  cert_auto_generate yes
+  flush_interval 30s
+  <user>
+    username moris
+    password foobar
+  </user>
+</match>

data/fluent-plugin-pull_forward.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-pull_forward"
+  spec.version       = "0.0.1"
+  spec.authors       = ["TAGOMORI Satoshi"]
+  spec.email         = ["tagomoris@gmail.com"]
+  spec.summary       = %q{Fluentd plugin to forward data, by pulling from input plugin}
+  spec.description   = %q{Fluentd plugin that store data to be forwarded, and send these when client(input plugin) requests it, over HTTPS and authentication}
+  spec.homepage      = "https://github.com/tagomoris/fluent-plugin-pull_forward"
+  spec.license       = "APLv2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "fluent-plugin-buffer-pullpool"
+  spec.add_runtime_dependency "fluent-mixin-config-placeholders"
+  spec.add_runtime_dependency "fluent-mixin-certificate"
+  spec.add_runtime_dependency "resolve-hostname", ">= 0.0.4"
+end

data/lib/fluent/plugin/in_pull_forward.rb

@@ -0,0 +1,118 @@
+module Fluent
+  class PullForwardInput < Input
+    DEFAULT_PULLFORWARD_LISTEN_PORT = 24280
+
+    Fluent::Plugin.register_input('pull_forward', self)
+
+    config_param :allow_self_signed_certificate, :bool, :default => true
+
+    config_param :fetch_interval, :time, :default => 600 # 10m
+    config_param :timeout, :time, :default => 60
+
+    config_section :server, param_name: :servers do
+      config_param :host, :string
+      config_param :port, :integer, :default => DEFAULT_PULLFORWARD_LISTEN_PORT
+      config_param :username, :string
+      config_param :password, :string
+    end
+
+    attr_reader :hostname_resolver
+
+    def initialize
+      super
+      require 'resolve/hostname'
+      require 'net/http'
+      require 'net/https'
+      require 'openssl'
+      require 'yajl'
+    end
+
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def configure(conf)
+      super
+
+      @verify_mode = if @allow_self_signed_certificate
+                       OpenSSL::SSL::VERIFY_NONE
+                     else
+                       OpenSSL::SSL::VERIFY_PEER
+                     end
+      @resolver = Resolve::Hostname.new(:system_resolver => true)
+    end
+
+    def start
+      super
+      @running = true
+      @thread = Thread.new(&method(:fetcher))
+    end
+
+    def shutdown
+      super
+      @running = false
+      @thread.join
+    end
+
+    def fetcher
+      next_fetch = Time.now
+      while @running
+        if Time.now >= next_fetch
+          @servers.each do |server|
+            if @running
+              fetch(server)
+            end
+          end
+          next_fetch = Time.now + @fetch_interval
+        end
+        break unless @running
+        sleep 1
+      end
+    end
+
+    def fetch(server)
+      body = nil
+
+      begin
+        address = @resolver.getaddress(server.host)
+        https = Net::HTTP.new(address, server.port)
+        https.open_timeout = @timeout
+        https.read_timeout = @timeout
+        https.use_ssl = true
+        https.verify_mode = @verify_mode
+
+        req = Net::HTTP::Get.new('/')
+        req.basic_auth server.username, server.password
+
+        res = https.start{ https.request(req) }
+        if res && res.is_a?(Net::HTTPSuccess)
+          body = res.body
+        else
+          log.warn "failed to GET from Fluentd PullForward: #{server.host}, #{address}:#{server.port}, by #{res.class}"
+        end
+      rescue IOError, EOFError, SystemCallError => e
+        log.warn "net/http GET raised an exception: #{e.class}, '#{e.message}'"
+      end
+      return unless body
+
+      data = nil
+      begin
+        data = Yajl::Parser.parse(body)
+      rescue => e
+        # maybe parse error
+        log.warn "an error occured for parse of transferred content: #{e.class}, '#{e.message}'"
+      end
+      return unless data
+
+      bundle = {}
+      data.each do |tag, time, record|
+        bundle[tag] ||= Fluent::MultiEventStream.new
+        bundle[tag].add(time, record)
+      end
+      bundle.each do |tag, es|
+        Fluent::Engine.emit_stream(tag, es)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_pull_forward.rb

@@ -0,0 +1,145 @@
+require 'fluent/mixin/config_placeholders'
+require 'fluent/mixin/certificate'
+require 'webrick'
+require 'webrick/https'
+
+require_relative 'webrick_logger_bridge'
+
+module Fluent
+  class PullForwardOutput < BufferedOutput
+    DEFAULT_PULLFORWARD_LISTEN_PORT = 24280
+
+    Fluent::Plugin.register_output('pull_forward', self)
+
+    config_param :self_hostname, :string
+    include Fluent::Mixin::ConfigPlaceholders
+
+    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :port, :integer, :default => DEFAULT_PULLFORWARD_LISTEN_PORT
+
+    config_param :server_loglevel, :string, :default => 'WARN'
+    config_param :auth_loglevel, :string, :default => 'FATAL'
+
+    config_set_default :buffer_type, 'pullpool'
+    config_set_default :flush_interval, 3600 # 1h
+
+    # REQUIRED: buffer_path
+
+    # same with TimeSlicedOutput + FileBuffer
+    # 16MB * 256 -> 4096MB
+    config_set_default :buffer_chunk_limit, 1024 * 1024 * 16 # 16MB
+    config_set_default :buffer_queue_limit, 256
+
+    include Fluent::Mixin::Certificate
+    # REQUIRED: self_hostname
+    # REQUIRED: 'cert_auto_generate yes' or 'cert_file_path PATH'
+
+    config_section :user, param_name: :users do
+      config_param :username, :string
+      config_param :password, :string
+    end
+
+    def initialize
+      super
+    end
+
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def configure(conf)
+      super
+      if @users.size < 1
+        raise Fluent::ConfigError, "no <user> sections specified"
+      end
+    end
+
+    def start
+      super
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @server.stop if @server
+      @thread.kill
+      @thread.join
+    end
+
+    class HtpasswdDummy < WEBrick::HTTPAuth::Htpasswd
+      # overwrite constructor NOT to generate htpasswd file on local filesystem
+      def initialize
+        @path = '/'
+        @mtime = Time.at(0)
+        @passwd = Hash.new
+        @auth_type = WEBrick::HTTPAuth::BasicAuth
+      end
+    end
+
+    def run
+      cert, key = self.certificate
+      realm = "Fluentd fluent-plugin-pullforward server"
+
+      logger = $log
+      auth_logger = Fluent::PluginLogger.new(logger)
+      auth_logger.level = @auth_loglevel
+      server_logger = Fluent::PluginLogger.new(logger)
+      server_logger.level = @server_loglevel
+
+      auth_db = HtpasswdDummy.new
+      @users.each do |user|
+        auth_db.set_passwd(realm, user.username, user.password)
+      end
+      authenticator = WEBrick::HTTPAuth::BasicAuth.new(
+        :UserDB => auth_db,
+        :Realm => realm,
+        :Logger => Fluent::PullForward::WEBrickLogger.new(auth_logger),
+      )
+
+      @server = WEBrick::HTTPServer.new(
+        :BindAddress => @bind,
+        :Port => @port,
+        # :DocumentRoot => '.',
+        :Logger => Fluent::PullForward::WEBrickLogger.new(server_logger),
+        :AccessLog => [],
+        :SSLEnable  => true,
+        :SSLCertificate => cert,
+        :SSLPrivateKey => key
+      )
+      @server.logger.info("hogepos")
+
+      @server.mount_proc('/') do |req, res|
+        unless req.ssl?
+          raise WEBrick::HTTPStatus::Forbidden, "pullforward plugin does not permit non-HTTPS requests"
+        end
+        if req.path != '/'
+          raise WEBrick::HTTPStatus::NotFound, "valid path is only '/'"
+        end
+        authenticator.authenticate(req, res)
+        res.content_type = 'application/json'
+        res.body = dequeue_chunks()
+      end
+
+      log.info "listening pullforward socket on #{@bind}:#{@port}"
+      @server.start
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def dequeue_chunks
+      response = []
+
+      unpacker = MessagePack::Unpacker.new
+
+      @buffer.pull_chunks do |chunk|
+        next if chunk.empty?
+        unpacker.feed_each(chunk.read) do |ary|
+          response << ary
+        end
+      end
+
+      response.to_json
+    end
+  end
+end

data/lib/fluent/plugin/webrick_logger_bridge.rb

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+module Fluent
+  module PullForward
+    class WEBrickLogger
+      FATAL = 1
+      ERROR = 2
+      WARN  = 3
+      INFO  = 4
+      DEBUG = 5
+
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def <<(str)
+        self.log(INFO, str.to_s)
+      end
+
+      def close
+        # NOP
+      end
+
+      def debug(msg)
+        self.log(DEBUG, msg)
+      end
+
+      def debug?
+        @logger.level > Fluent::Log::LEVEL_TRACE
+      end
+
+      def error(msg)
+        self.log(ERROR, msg)
+      end
+
+      def error?
+        @logger.level > Fluent::Log::LEVEL_WARN
+      end
+
+      def fatal(msg)
+        self.log(FATAL, msg)
+      end
+
+      def fatal?
+        @logger.level > Fluent::Log::LEVEL_ERROR
+      end
+
+      def info(msg)
+        self.log(INFO, msg)
+      end
+
+      def info?
+        @logger.level > Fluent::Log::LEVEL_DEBUG
+      end
+
+      def level
+        # (Fluentd logger level num) -> (Webrick level num)
+        # 5 -> 1
+        # 4 -> 2
+        # 3 -> 3
+        # 2 -> 4
+        # 1 -> 5
+        # (6 - level)
+        6 - @logger.level
+      end
+
+      def level=(lv)
+        @logger.level = case lv
+                        when FATAL then 'fatal'
+                        when ERROR then 'error'
+                        when WARN then 'warn'
+                        when INFO then 'info'
+                        when DEBUG then 'debug'
+                        else
+                          raise ArgumentError, "Invalid loglevel for webrick bridge logger: #{lv}"
+                        end
+      end
+
+      def log(level, msg)
+        case level
+        when FATAL
+          @logger.fatal(msg)
+        when ERROR
+          @logger.error(msg)
+        when WARN
+          @logger.warn(msg)
+        when INFO
+          @logger.info(msg)
+        when DEBUG
+          @logger.debug(msg)
+        else
+          raise ArgumentError, "Invalid loglevel for webrick bridge logger: #{lv}"
+        end
+      end
+
+      def warn(msg)
+        self.log(WARN, msg)
+      end
+
+      def warn?
+        @logger.level > Fluent::Log::LEVEL_INFO
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_pull_forward'
+require 'fluent/plugin/out_pull_forward'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_pull_forward.rb

@@ -0,0 +1,5 @@
+require 'helper'
+
+class PullForwardInputTest < Test::Unit::TestCase
+  # check syntax error only now...
+end

data/test/plugin/test_out_pull_forward.rb

@@ -0,0 +1,5 @@
+require 'helper'
+
+class PullForwardOutputTest < Test::Unit::TestCase
+  # check syntax error only now...
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-pull_forward
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- TAGOMORI Satoshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-buffer-pullpool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-certificate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resolve-hostname
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+description: Fluentd plugin that store data to be forwarded, and send these when client(input
+  plugin) requests it, over HTTPS and authentication
+email:
+- tagomoris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- example/simple_in.conf
+- example/simple_out.conf
+- fluent-plugin-pull_forward.gemspec
+- lib/fluent/plugin/in_pull_forward.rb
+- lib/fluent/plugin/out_pull_forward.rb
+- lib/fluent/plugin/webrick_logger_bridge.rb
+- misc/plugin_image.png
+- test/helper.rb
+- test/plugin/test_in_pull_forward.rb
+- test/plugin/test_out_pull_forward.rb
+homepage: https://github.com/tagomoris/fluent-plugin-pull_forward
+licenses:
+- APLv2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to forward data, by pulling from input plugin
+test_files:
+- test/helper.rb
+- test/plugin/test_in_pull_forward.rb
+- test/plugin/test_out_pull_forward.rb
+has_rdoc: