fluent-plugin-beats 0.1.4 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: '092ac019d242807f0de4fb50f8f79947057b2084de286c1e57bfd38d5e9149c0'
-  data.tar.gz: ff608d2612acf8e3d0f23c6d8aa798d0ae301e94b3394faa630075d9841a90b4
+SHA1:
+  metadata.gz: 73b84f27924633682deed6e9a10728e152b48a7d
+  data.tar.gz: 9d943e19a7a44fff61ff4e30bb1af64340552e67
 SHA512:
-  metadata.gz: dc87c07cdcf60433a5b11fed9259b63e215045514f6c23e3e57a10cd49be1d36d9abb6749149bea1a739fc3d25724d51675c7ae72d13a8f813d9d5ce0f2528a2
-  data.tar.gz: f53064e29c5ddb2663b1ee0f64d8c1641fbf0c83670b6e5485bc5312b48c48d55c57eeb49b5fc731ed862755fc6494295c447a22774030a991b25f09085b7984
+  metadata.gz: c81ee764955868ff64a2565c3c0a41e07fdda4a607c820383d615667121edbcbc10274bb864b41775cea7ccac5b1ed703e04b5e704d4708900e8837496788000
+  data.tar.gz: 3dd3d1d206f46bfd93fb4fb9bf81d8a6eab6a4f0f98b25c5ab76220d0a5f346ac04fc45edf0d18a6738d3d498ee6770fbe281515a19c7caa1f42a8546f45a37e

data/README.md

@@ -4,9 +4,16 @@
 
 This plugin uses lumberjack protocol for communicating with each beat.
 
+## Requirements
+
+| fluent-plugin-beats | fluentd | ruby |
+|-------------------|---------|------|
+| >= 1.0.0 | >= v1.0.0 | >= 2.1 |
+|  < 1.0.0 | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
-    $ gem install fluent-plugin-beats
+    $ gem install fluent-plugin-beats --no-document
 
 ## Configuration
 
@@ -29,6 +36,8 @@ Configuration example:
 
   The port to listen to. Default Value is `5044`.
 
+  If you use this plugin under multi-process environment in v1, the plugin will be launched in each worker. Port is assigned in sequential number, e.g. 5044, 5045 ... 504N.
+
 **bind**
 
   The bind address to listen to. Default Value is 0.0.0.0 (all addresses)

data/VERSION

@@ -1 +1 @@
-0.1.4
+1.0.0

data/lib/fluent/plugin/in_beats.rb

@@ -15,44 +15,57 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+require "lumberjack/beats"
+require "lumberjack/beats/server"
+require 'concurrent/executor/cached_thread_pool'
 
-require 'fluent/input'
-require 'fluent/parser'
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+require 'fluent/time'
 
-module Fluent
+module Fluent::Plugin
   class BeatsInput < Input
-    Plugin.register_input('beats', self)
+    Fluent::Plugin.register_input('beats', self)
 
-    def initialize
-      super
+    include Fluent::TimeMixin::Parser
 
-      require "lumberjack/beats"
-      require "lumberjack/beats/server"
-      require 'concurrent/executor/cached_thread_pool'
-    end
+    helpers :compat_parameters, :parser, :thread
+
+    DEFAULT_PARSER = 'none'.freeze
 
     config_param :port, :integer, :default => 5044
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :tag, :string, :default => nil
     config_param :metadata_as_tag, :bool, :default => nil
-    config_param :format, :string, :default => nil
     config_param :max_connections, :integer, :default => nil # CachedThreadPool can't limit the number of threads
     config_param :use_ssl, :string, :default => false
     config_param :ssl_certificate, :string, :default => nil
     config_param :ssl_key, :string, :default => nil
     config_param :ssl_key_passphrase, :string, :default => nil
 
+    config_section :parse do
+      config_set_default :@type, DEFAULT_PARSER
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
       super
 
       if !@tag && !@metadata_as_tag
-        raise ConfigError,  "'tag' or 'metadata_as_tag' parameter is required on beats input"
+        raise Fluent::ConfigError,  "'tag' or 'metadata_as_tag' parameter is required on beats input"
       end
 
-      @time_parser = Fluent::TextParser::TimeParser.new('%Y-%m-%dT%H:%M:%S.%N%z')
-      if @format
-        @parser = Plugin.new_parser(@format)
-        @parser.configure(conf)
+      @port += fluentd_worker_id
+      @time_parser = time_parser_create(format: '%Y-%m-%dT%H:%M:%S.%N%z')
+
+      @parser_config = conf.elements('parse').first
+      if @parser_config
+        @parser = parser_create
       end
       @connections = []
     end
@@ -66,13 +79,12 @@ module Fluent
       # Lumberjack::Beats::Server depends on normal accept so we need to launch thread for each connection.
       # TODO: Re-implement Beats Server with Cool.io for resource control
       @thread_pool = Concurrent::CachedThreadPool.new(:idletime => 15) # idletime setting is based on logstash beats input
-      @thread = Thread.new(&method(:run))
+      thread_create(:in_beats_runner, &method(:run))
     end
 
     def shutdown
       @lumberjack.close rescue nil
       @thread_pool.shutdown
-      @thread.join
 
       super
     end
@@ -88,7 +100,7 @@ module Fluent
             conn.close # close for retry on beats side
             sleep 1
             next
-          end          
+          end
           @connections << conn
         end
 
@@ -97,7 +109,7 @@ module Fluent
             conn.run { |map|
               tag = @metadata_as_tag ? map['@metadata']['beat'] : @tag
 
-              if map.has_key?('message') && @format
+              if map.has_key?('message') && @parser_config
                 message = map.delete('message')
                 @parser.parse(message) { |time, record|
                   record['@timestamp'] = map['@timestamp']

data/test/plugin/test_in_beats.rb

@@ -1,4 +1,5 @@
 require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_beats'
 require 'lumberjack/beats/client'
 require 'helper'
@@ -18,7 +19,7 @@ class BeatsInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::BeatsInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::BeatsInput).configure(conf)
   end
 
   def create_client
@@ -52,11 +53,11 @@ class BeatsInputTest < Test::Unit::TestCase
       client.write(payload)
     end
 
-    tag, es = d.emit_streams[0]
-    assert_equal tag, 'test.beats'
+    es = d.events
+    assert_equal es[0][0], 'test.beats'
     assert_equal es.length, 1
 
-    time, record = es[0]
+    tag, time, record = es[0]
     assert_equal time.to_i, Time.parse(timestamp).to_i
     assert_equal record, payload
   end
@@ -76,12 +77,43 @@ class BeatsInputTest < Test::Unit::TestCase
       client.write(payload)
     end
 
-    tag, es = d.emit_streams[0]
-    assert_equal tag, 'heartbeat'
+    es = d.events
+    assert_equal es[0][0], 'heartbeat'
     assert_equal es.length, 1
 
-    time, record = es[0]
+    tag, time, record = es[0]
     assert_equal time.to_i, Time.parse(timestamp).to_i
     assert_equal record, payload
   end
+
+  def test_input_as_json
+    d = create_driver(CONFIG + %[format json])
+
+    timestamp = '2018-01-01T12:30:00.000Z'
+    payload = {
+        '@metadata' => {'beat' => 'heartbeat'},
+        '@timestamp' => timestamp,
+        'message' => '{"msg":"as_json"}'
+    }
+    expected = {
+        '@metadata' => {'beat' => 'heartbeat'},
+        '@timestamp' => timestamp,
+        'msg' => 'as_json'
+    }
+
+    now = Fluent::Engine.now
+    d.run do
+      client = create_client
+      client.write(payload)
+    end
+
+    es = d.events
+    assert_equal es[0][0], 'test.beats'
+    assert_equal es.length, 1
+
+    _tag, time, record = es[0]
+    assert_in_delta now, time, 1.0
+    assert_equal record["@timestamp"], timestamp
+    assert_equal record, expected
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-beats
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 1.0.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-21 00:00:00.000000000 Z
+date: 2018-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: Elastic beats plugin for Fluentd event collector