fluent-plugin-wendelin 0.1 → 0.5

data/README.md

@@ -12,7 +12,7 @@ Configuration
 
 A sample configuration to setup ingestion of data for a tag to Wendelin:
 
-```apache
+```
 <match your.sensor.tag>
   @type wendelin
   @id wendelin_out
@@ -20,6 +20,7 @@ A sample configuration to setup ingestion of data for a tag to Wendelin:
   streamtool_uri    <Wendelin_URL>/erp5/portal_ingestion_policies/<YOUR_INGESTION_POLICY_ID>
   user      <your_wendelin_user>
   password  <your_wendelin_password>
+  use_keep_alive  <true or false>
 
   # all parameters of BufferedOutput & Output classes are supported too, e.g.
   # `buffer_type`, `flush_interval`, `num_threads`, `log_level`, etc - see
@@ -43,7 +44,6 @@ See *example/to_wendelin.conf* for fully setup example.
 TODO
 ----
 
-- make content to be posted over HTTP as raw (instead of urlencode data).
 - HTTPS certificates are currently not verified.
 - X.509 certificates instead of user / password.
 - cleanup

data/example/to_wendelin.conf

@@ -1,14 +1,5 @@
 # Example configuration of Wendelin output for Fluentd
 
-## built-in TCP input, so we can accept messages from other fluentd(s) and so
-## that fluent-cat works:
-## $ echo <json|msgpack> | fluent-cat <tag>
-<source>
-  @type forward
-  @id forward_input
-</source>
-
-
 # HTTP input
 # http://localhost:8888/<tag>?json=<json>
 <source>
@@ -17,74 +8,28 @@
   port 8888
 </source>
 
-
-## output tag=sensor.** to Wendelin
-<match <YOUR_SENSOR_TAG>.**>
-  @type wendelin
+# This will match all HTTP request like so
+# ivan@debian: ~$ curl -X POST -d 'json={"foo1":"bar1"}' http://localhost:8888/test_sensor.test_product
+# then send to Wendelin and append as MsgPack encoded string at Wendelin side 
+<match test_sensor.test_product.**>
+  @type             wendelin
   @id wendelin_out
-
-  streamtool_uri    <Wendelin_URL>/erp5/portal_ingestion_policies/<YOUR_INGESTION_POLICY_ID>
-  # TODO ^^^ do not check peer's certificate
-  user      <your_wendelin_user>
-  password  <your_wendelin_password>
-
-  # all parameters of BufferedOutput & Output classes are supported too, e.g.
-  # `buffer_type`, `flush_interval`, `num_threads`, `log_level`, etc - see
-  # their code near:
-  #
-  #   https://github.com/fluent/fluentd/blob/master/lib/fluent/output.rb#L182
-  #
-  # logging setup description:
-  #
-  #   http://docs.fluentd.org/articles/logging
-
+  streamtool_uri    <Wendelin_URL>/erp5/portal_ingestion_policies/default   
+  user              <your_wendelin_user>
+  password          <your_wendelin_password>
+  use_keep_alive    true  
   buffer_type       memory
-  #buffer_path       "#{ENV['HOME']/var}"
   flush_interval    5s
 </match>
 
-
-
-
-# ---- monitoring & debugging ----
-
-
-# Listen HTTP for monitoring
-# http://localhost:24220/api/plugins
-# http://localhost:24220/api/plugins?type=TYPE
-# http://localhost:24220/api/plugins?tag=MYTAG
-<source>
-  @type monitor_agent
-  @id monitor_agent_input
-
-  port 24220
-</source>
-
-# Listen DRb for debug
-<source>
-  @type debug_agent
-  @id debug_agent_input
-
-  bind 127.0.0.1
-  port 24230
-</source>
-
-
-## match tag=debug.** and dump to console
-<match debug.**>
-  @type stdout
-  @id stdout_output
-</match>
-
-
-## match fluent's internal events
-#<match fluent.**>
-#  @type null
-#</match>
-
-## match not matched logs and write to file
-#<match **>
-#  @type file
-#  path /var/log/fluent/else
-#  compress gz
-#</match>
+# Example: tailing existing file and send its contents to Wendelin
+# Uncomment below to enable
+#<source>
+#  @type             tail
+#  path              test.log
+#  pos_file          test.log.pos
+#  tag               test_sensor.test_product
+#  <parse>
+#    @type none
+#  </parse>
+#</source>

data/fluent-plugin-wendelin.gemspec

@@ -1,17 +1,17 @@
 Gem::Specification.new do |gem|
     gem.name            = "fluent-plugin-wendelin"
-    gem.version         = "0.1"
-    gem.authors         = ["Kirill Smelkov"]
-    gem.email           = ["kirr@nexedi.com"]
+    gem.version         = "0.5"
+    gem.authors         = ["Nexedians"]
+    gem.email           = ["klaus@nexedi.com"]
     gem.summary         = %q{Fluentd output to Wendelin}
     gem.description     = %q{Fluentd output plugin to forward data to Wendelin system}
     gem.homepage        = "https://lab.nexedi.com/nexedi/fluent-plugin-wendelin"
-    gem.license         = "APLv2"
+    gem.license         = "GPLv3+ with wide exception for Open-Source"
 
     gem.files           = `git ls-files -z`.split("\x0")
     gem.require_paths   = ["lib"]
 
-    gem.add_runtime_dependency "fluentd", "~> 0"
+    gem.add_runtime_dependency "fluentd", "< 2"
     # ? net/http
     # ? openssl
 end

data/lib/fluent/plugin/out_wendelin.rb

@@ -2,31 +2,35 @@
 # Copyright (C) 2015  Nexedi SA and Contributors.
 #                     Kirill Smelkov <kirr@nexedi.com>
 #
-#   This program is free software: you can Use, Study, Modify and Redistribute
-#   it under the terms of the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
 #
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
 
 
 # This plugin hooks into Fluentd in a way similiar to out_forward and
 # out_secure_forward and outputs event stream to a Wendelin HTTP endpoint.
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 require_relative 'wendelin_client'
 
 
-module Fluent
+module Fluent::Plugin
 
-  class WendelinOutput < ObjectBufferedOutput   # XXX verify base class
-    Plugin.register_output('wendelin', self)
+  class WendelinOutput < Output
+    Fluent::Plugin.register_output('wendelin', self)
 
     # where Wendelin's Input Stream Tool is located,
     # ex http://example.com/erp5/portal_ingestion_policies/example_ingestion
@@ -38,16 +42,32 @@ module Fluent
     config_param    :user,      :string,  :default => nil
     config_param    :password,  :string,  :default => nil
 
-
+    config_param    :use_keep_alive,  :bool,  :default => false
+
+    config_param    :ssl_timeout,        :integer,  :default => 60
+    config_param    :open_timeout,       :integer,  :default => 60
+    config_param    :read_timeout,       :integer,  :default => 60
+    config_param    :keep_alive_timeout, :integer,  :default => 300
+	
+	config_section :buffer do
+	  config_set_default :chunk_keys, ["tag"]
+	end
+	
     def configure(conf)
         super
+		
+		unless @chunk_key_tag
+		  raise Fluent::ConfigError, "buffer chunk key must include 'tag' for wendelin output"
+		end
 
         credentials = {}
         if @user
             credentials['user']     = @user
             credentials['password'] = @password
         end
-        @wendelin = WendelinClient.new(@streamtool_uri, credentials)
+        @wendelin = WendelinClient.new(@streamtool_uri, credentials, @log,
+                                       @ssl_timeout, @open_timeout,
+                                       @read_timeout, @keep_alive_timeout)
     end
 
     def start
@@ -60,12 +80,11 @@ module Fluent
         # TODO
     end
 
-
-    # hooked to ObjectBufferedOutput - write out records from a buffer chunk for a tag.
+    # Use normal "Synchronous Buffer" - write out records from a buffer chunk for a tag.
     #
-    # NOTE this is called from a separate thread (see OutputThread and BufferedOutput)
-    def write_objects(tag, chunk)
-        # NOTE if this fail and raises -> it will unroll to BufferedOutput#try_flush
+    def write(chunk)
+	    return if chunk.empty?
+        # NOTE if this fail and raises -> it will unroll to Output#try_flush
         # which detects errors and retries outputting logs up to retry maxcount
         # times and aborts outputting current logs if all try fail.
         #
@@ -78,9 +97,13 @@ module Fluent
 
         # for input_stream_ref use tag as-is - it will be processed/translated
         # further on server by Wendelin
-        reference = tag
+        reference = chunk.metadata.tag
 
-        @wendelin.ingest(reference, data_chunk)
+        if @use_keep_alive
+          @wendelin.ingest_with_keep_alive(reference, data_chunk)
+        else
+          @wendelin.ingest(reference, data_chunk)
+        end
     end
 
   end

data/lib/fluent/plugin/out_wendelin_formatted.rb

@@ -0,0 +1,57 @@
+# Fluentd output plugin to ingest data to Wendelin system
+# Copyright (C) 2015  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# This plugin hooks into Fluentd in a way similiar to out_forward and
+# out_secure_forward and outputs event stream to a Wendelin HTTP endpoint.
+
+require 'fluent/plugin/formatter'
+require_relative 'out_wendelin'
+
+
+module Fluent::Plugin  
+  class WendelinOutputFormatted < WendelinOutput
+    Fluent::Plugin.register_output('wendelin_formatted', self)
+	
+	helpers :formatter
+	
+	config_section :format do
+	  config_set_default :@type, 'single_value'
+	end
+	
+    def configure(conf)
+        super
+		@formatter = formatter_create
+	end
+
+    def formatted_to_msgpack_binary?
+      @formatter_configs.first[:@type] == 'msgpack'
+    end
+
+	def format(tag, time, record)
+        @formatter.format(tag, time, record)
+	end
+	
+	def write(chunk)
+	  super
+	end
+  
+  end
+
+end

data/lib/fluent/plugin/wendelin_client.rb

@@ -2,97 +2,190 @@
 # Copyright (C) 2015  Nexedi SA and Contributors.
 #                     Kirill Smelkov <kirr@nexedi.com>
 #
-#   This program is free software: you can Use, Study, Modify and Redistribute
-#   it under the terms of the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
 #
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
 
 require 'net/http'
 require 'openssl'
 
-# show request/response traces or not
-$trace    = false
-
-
 # class representing a Wendelin client
 class WendelinClient
 
   # `streamtool_uri` - URI pointing to portal_input_data_stream "mountpoint"
   # `credentials`    # {'user' => _, 'password' => _}     TODO change to certificate
-  def initialize(streamtool_uri, credentials)
-    @streamtool_uri = streamtool_uri
-    @credentials    = credentials
+  # `log` - logger to use
+  def initialize(streamtool_uri, credentials, log,
+                 ssl_timeout, open_timeout, read_timeout, keep_alive_timeout)
+    @streamtool_uri     = streamtool_uri
+    @credentials        = credentials
+    @log                = log
+    @ssl_timeout        = ssl_timeout
+    @open_timeout       = open_timeout
+    @read_timeout       = read_timeout
+    @keep_alive_timeout = keep_alive_timeout
   end
 
 
+  # start request in an independent function to keep the connection open
+  def start_connection(uri)
+
+    @log.debug "start new connection"
+
+    @http = Net::HTTP.start(uri.hostname, uri.port,
+      :use_ssl => (uri.scheme == 'https'),
+      :verify_mode  => OpenSSL::SSL::VERIFY_NONE,
+
+      # Net::HTTP default open timeout is infinity, which results
+      # in thread hang forever if other side does not fully
+      # establish connection. Default read_timeout is 60 seconds.
+      # We go safe way and make sure all timeouts are defined.
+      :ssl_timeout  => @ssl_timeout,
+      :open_timeout => @open_timeout,
+      :read_timeout => @read_timeout,
+      :keep_alive_timeout => @keep_alive_timeout,)
+  end
+
   # ingest `data_chunk` to a stream referenced as `reference`
-  def ingest(reference, data_chunk)
-      uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
-      req = Net::HTTP::Post.new(uri)
-      if @credentials.has_key?('user')
-        req.basic_auth @credentials['user'], @credentials['password']
-      end
-
-      # TODO ensure content-type is 'raw', e.g. this way
-      # (but then querystring ?reference=... is lost)
-      # req.body = data_chunk
-      # req.content_type = 'application/octet-stream'
-      req.set_form_data('data_chunk' => data_chunk)
-
-      if $trace
-          puts '>>> REQUEST'
-          puts "method\t=> #{req.method}"
-          puts "path\t=> #{req.path}"
-          puts "uri\t=> #{req.uri}"
-          puts "body\t=> #{req.body}"
-          puts "body_stream\t=> #{req.body_stream}"
-          req.each {|h| puts "#{h}:\t#{req[h]}"}
-          puts
-      end
-
-      begin
-          # TODO keep connection open (so that every new ingest does not do
-          # full connect again)
-          res = Net::HTTP.start(uri.hostname, uri.port,
-                      :use_ssl      => (uri.scheme == 'https'),
-                      # NOTE = "do not check server cert"
-                      # TODO move this out to conf parameters
-                      :verify_mode  => OpenSSL::SSL::VERIFY_NONE
-                ) do |http|
-              http.request(req)
-          end
-
-      rescue
-          # some http/ssl/other connection error
-          puts "HTTP ERROR:"
-          raise
-
-      else
-          if $trace
-              puts '>>> RESPONSE'
-              res.each {|h| puts "#{h}:\t#{res[h]}"}
-              puts "code\t=> #{res.code}"
-              puts "msg\t=> #{res.message}"
-              puts "class\t=> #{res.class}"
-              puts "body:", res.body
-          end
-
-          if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
-              #puts "ingested ok"
-          else
-              puts "FAIL:"
-              res.value
-          end
-      end
+  def ingest_with_keep_alive(reference, data_chunk)
+    uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
+    # call start_connection if http is undefined
+    if ! defined? @http
+      start_connection(uri)
+    end
+
+    # connect again if the connection is not started
+    if ! @http.started?()
+      start_connection(uri)
+    end
+
+    @request = Net::HTTP::Post.new(uri)
+
+    # When using 'application/x-www-form-urlencoded', Ruby encodes with regex
+    # and it is far too slow. Such POST is legit:
+    # https://stackoverflow.com/a/14710450
+    @request.body = data_chunk
+    @request.content_type = 'application/octet-stream'
+
+    if @credentials.has_key?('user')
+      @request.basic_auth @credentials['user'], @credentials['password']
+    end
+
+    @log.on_trace do
+      @log.trace '>>> REQUEST'
+      @log.trace "method\t=> #{@request.method}"
+      @log.trace "path\t=> #{@request.path}"
+      @log.trace "uri\t=> #{@request.uri}"
+      @log.trace "body\t=> #{@request.body}"
+      @log.trace "body_stream\t=> #{@request.body_stream}"
+      @request.each {|h| @log.trace "#{h}:\t#{@request[h]}"}
+      @log.trace
+    end
+
+    begin
+      res = @http.request(@request) # Net::HTTPResponse object
+    end
+
+    rescue
+      # some http/ssl/other connection error
+      @log.warn "HTTP ERROR:"
+    raise
+  else
+    @log.on_trace do
+      @log.trace '>>> RESPONSE'
+      res.each {|h| @log.trace "#{h}:\t#{res[h]}"}
+      @log.trace "code\t=> #{res.code}"
+      @log.trace "msg\t=> #{res.message}"
+      @log.trace "class\t=> #{res.class}"
+      @log.trace "body:", res.body
+    end
+
+    if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
+      #@log.info "ingested ok"
+    else
+      @log.warn "FAIL:"
+      res.value
+    end
   end
 
+  def ingest(reference, data_chunk)
+     uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
+     req = Net::HTTP::Post.new(uri)
+     if @credentials.has_key?('user')
+       req.basic_auth @credentials['user'], @credentials['password']
+     end
+
+     # When using 'application/x-www-form-urlencoded', Ruby encodes with regex
+     # and it is far too slow. Such POST is legit:
+     # https://stackoverflow.com/a/14710450
+     req.body = data_chunk
+     req.content_type = 'application/octet-stream'
+
+     @log.on_trace do
+         @log.trace '>>> REQUEST'
+         @log.trace "method\t=> #{req.method}"
+         @log.trace "path\t=> #{req.path}"
+         @log.trace "uri\t=> #{req.uri}"
+         @log.trace "body\t=> #{req.body}"
+         @log.trace "body_stream\t=> #{req.body_stream}"
+         req.each {|h| @log.trace "#{h}:\t#{req[h]}"}
+         @log.trace
+     end
+
+     begin
+         # TODO keep connection open (so that every new ingest does not do
+         # full connect again)
+         res = Net::HTTP.start(uri.hostname, uri.port,
+                     :use_ssl      => (uri.scheme == 'https'),
+                     # NOTE = "do not check server cert"
+                     # TODO move this out to conf parameters
+                     :verify_mode  => OpenSSL::SSL::VERIFY_NONE,
+
+                     # Net::HTTP default open timeout is infinity, which results
+                     # in thread hang forever if other side does not fully
+                     # establish connection. Default read_timeout is 60 seconds.
+                     # We go safe way and make sure all timeouts are defined.
+                     :ssl_timeout  => @ssl_timeout,
+                     :open_timeout => @open_timeout,
+                     :read_timeout => @read_timeout,
+               ) do |http|
+             http.request(req)
+         end
+
+     rescue
+         # some http/ssl/other connection error
+         @log.warn "HTTP ERROR:"
+         raise
+
+     else
+         @log.on_trace do
+             @log.trace '>>> RESPONSE'
+             res.each {|h| @log.trace "#{h}:\t#{res[h]}"}
+             @log.trace "code\t=> #{res.code}"
+             @log.trace "msg\t=> #{res.message}"
+             @log.trace "class\t=> #{res.class}"
+             @log.trace "body:", res.body
+         end
+
+         if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
+             #@log.info "ingested ok"
+         else
+             @log.warn "FAIL:"
+             res.value
+         end
+     end
+ end
+
 end

data/test/wendelin_test.rb

@@ -5,6 +5,10 @@
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '../lib'))
 require 'fluent/plugin/wendelin_client'
+require 'fluent/log'
+require 'fluent/engine' # for log
+
+log       = Fluent::Log.new(out=STDERR, level=Fluent::Log::LEVEL_TRACE)
 
 # FIXME hardcoded
 $user     = 'user'
@@ -14,7 +18,7 @@ $password = 'password'
 $erp5       = "https://example.com/erp5"                # erp5 root
 $streamtool = "#{$erp5}/portal_input_data_streams"      # where Input Stream Tool is mounted
 
-$wendelin   = WendelinClient.new($streamtool, {'user' => $user, 'password' => $password})
+$wendelin   = WendelinClient.new($streamtool, {'user' => $user, 'password' => $password}, log)
 
 
 def ingest(input_stream_ref, data_chunk)