fluent-plugin-wire-protocol-compat 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a48a636525c3ed60d88189bbd630a2cc3e0b17b
+  data.tar.gz: 3f95a2c5e718392dc86aa49d2d4fb56e43b4ba35
+SHA512:
+  metadata.gz: 794d3431ce9fffab468bad0b1455c0c284246f2461732b1f44b6638e202e6e34a576175e10e202ff46fcfbdffad956d1115e64cbd932c8084517fce484bba71d
+  data.tar.gz: 00d181daed6424ba79e696dd03d1b39013f0986a3e49ec7922e2901cbb0da51846b9483b84618186c3f6f06bf232938a2e853d33a69d5c0c2cc9a775486a6763

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.swp
+*.swo
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.2.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-wire-protocol-compat.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Fluent::WireProtocolCompatInput
+
+This plugin makes the wire protocol for [in\_udp](http://docs.fluentd.org/articles/in_udp) and [in\_tcp](http://docs.fluentd.org/articles/in_tcp) compatible with [in\_forward](http://docs.fluentd.org/articles/in_forward#protocol)'s.
+
+Note: You need to install [fluent-plugin-msgpack-parser](https://rubygems.org/gems/fluent-plugin-msgpack-parser) to add [msgpack](http://msgpack.org/) support for `in_udp` and `in_tcp`.
+
+## Installation
+
+Please check Fluentd's [plugin management docs](http://docs.fluentd.org/articles/plugin-management)
+
+## Usage
+### Example configuration
+```
+<source>
+  type wire_protocol_compat
+  <input>
+    type udp
+    port 12201
+    bind 127.0.0.1
+    format json
+    tag some_tag
+  </input>
+</source>
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/bitex-la/fluent-plugin-wire-protocol-compat/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## TODO
+* Additional formats specs
+* Extract rspec-fluentd gem

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-wire-protocol-compat.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-wire-protocol-compat"
+  spec.version       = File.read("VERSION").strip
+  spec.authors       = ["Tomás Rojas"]
+  spec.email         = ["tmsrjs@gmail.com"]
+
+  spec.summary       = %q{Adds in_forward wire protocol support to in_udp and in_tcp}
+  spec.homepage      = "https://github.com/bitex-la/fluent-plugin-wire-protocol-compat"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", ">= 0.10.51"
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "timecop"
+end

data/lib/fluent/plugin/in_wire_protocol_compat.rb

@@ -0,0 +1,109 @@
+require 'fluent/input'
+require 'fluent/plugin'
+
+module Fluent
+  class WireProtocolCompatInput < Input
+    Plugin.register_input('wire_protocol_compat', self)
+    def configure(conf)
+      super
+
+      @inputs = [ ]
+      config.each_element('input') do |input_config|
+        type = input_config.delete('type')
+        next unless [ 'udp', 'tcp' ].include? type
+        base_input = Plugin.new_input(type).class
+        input_klass = Class.new(base_input) do
+          include WireProtocolCompatInput::SocketUtilBaseInputMethods
+        end
+        input = input_klass.new
+        input.configure(input_config)
+        if input.format == 'json'
+          parser_klass = Class.new(Fluent::TextParser::JSONParser) do
+            include WireProtocolCompatInput::JSONParserMethods
+          end
+          input.instance_eval do
+            @parser = parser_klass.new
+            @parser.configure(@config)
+          end
+        end
+        @inputs << input
+      end
+    end
+
+    def start
+      super
+      @inputs.each(&:start)
+    end
+
+    def shutdown
+      super
+      @inputs.each(&:shutdown)
+    end
+
+    module JSONParserMethods
+      def normalize_hash(record)
+        value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
+        if value
+          if @time_format
+            time = @mutex.synchronize { @time_parser.parse(value) }
+          else
+            begin
+              time = value.to_i
+            rescue => e
+              raise ParserError, "invalid time value: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+            end
+          end
+        else
+          if @estimate_current_event
+            time = Engine.now
+          else
+            time = nil
+          end
+        end
+        [ time, record ]
+      end
+
+      def parse(text)
+        parsed = @load_proc.call(text)
+        case parsed
+        when Hash
+          normalized = normalize_hash(parsed)
+        when Array
+          normalized = parsed
+        end
+        yield *normalized
+      end
+    end
+
+    module SocketUtilBaseInputMethods
+      def on_message(msg, addr)
+        @parser.parse(msg) do |*parsed|
+          case parsed[0]
+          when String
+            tag, time, record = parsed
+            records = time.is_a?(Array) ? time : [ [ time, record ] ]
+          when Integer
+            tag = @tag
+            records = [ parsed ]
+          end
+
+          time, record = records.first
+          unless time && record
+            log.warn "pattern not match: #{msg.inspect}"
+            return
+          end
+
+          es = MultiEventStream.new
+          records.each do |time, record|
+            record[@source_host_key] = addr[3] if @source_host_key
+            es.add(time, record)
+          end
+          router.emit_stream(tag, es)
+        end
+      rescue => e
+        log.error msg.dump, :error => e, :error_class => e.class, :host => addr[3]
+        log.error_backtrace
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-wire-protocol-compat
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tomás Rojas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.51
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.51
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- tmsrjs@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-wire-protocol-compat.gemspec
+- lib/fluent/plugin/in_wire_protocol_compat.rb
+homepage: https://github.com/bitex-la/fluent-plugin-wire-protocol-compat
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Adds in_forward wire protocol support to in_udp and in_tcp
+test_files: []