fluent-plugin-wendelin 0.3 → 0.4

data/README.md

@@ -12,7 +12,7 @@ Configuration
 
 A sample configuration to setup ingestion of data for a tag to Wendelin:
 
-```apache
+```
 <match your.sensor.tag>
   @type wendelin
   @id wendelin_out
@@ -20,6 +20,7 @@ A sample configuration to setup ingestion of data for a tag to Wendelin:
   streamtool_uri    <Wendelin_URL>/erp5/portal_ingestion_policies/<YOUR_INGESTION_POLICY_ID>
   user      <your_wendelin_user>
   password  <your_wendelin_password>
+  use_keep_alive  <true or false>
 
   # all parameters of BufferedOutput & Output classes are supported too, e.g.
   # `buffer_type`, `flush_interval`, `num_threads`, `log_level`, etc - see
@@ -43,7 +44,6 @@ See *example/to_wendelin.conf* for fully setup example.
 TODO
 ----
 
-- make content to be posted over HTTP as raw (instead of urlencode data).
 - HTTPS certificates are currently not verified.
 - X.509 certificates instead of user / password.
 - cleanup

data/example/to_wendelin.conf

@@ -28,6 +28,8 @@
   user      <your_wendelin_user>
   password  <your_wendelin_password>
 
+  use_keep_alive true  
+
   # all parameters of BufferedOutput & Output classes are supported too, e.g.
   # `buffer_type`, `flush_interval`, `num_threads`, `log_level`, etc - see
   # their code near:

data/fluent-plugin-wendelin.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |gem|
     gem.name            = "fluent-plugin-wendelin"
-    gem.version         = "0.3"
-    gem.authors         = ["Kirill Smelkov"]
-    gem.email           = ["kirr@nexedi.com"]
+    gem.version         = "0.4"
+    gem.authors         = ["Nexedians"]
+    gem.email           = ["klaus@nexedi.com"]
     gem.summary         = %q{Fluentd output to Wendelin}
     gem.description     = %q{Fluentd output plugin to forward data to Wendelin system}
     gem.homepage        = "https://lab.nexedi.com/nexedi/fluent-plugin-wendelin"

data/lib/fluent/plugin/out_wendelin.rb

@@ -38,6 +38,12 @@ module Fluent
     config_param    :user,      :string,  :default => nil
     config_param    :password,  :string,  :default => nil
 
+    config_param    :use_keep_alive,  :bool,  :default => false
+
+    config_param    :ssl_timeout,        :integer,  :default => 60
+    config_param    :open_timeout,       :integer,  :default => 60
+    config_param    :read_timeout,       :integer,  :default => 60
+    config_param    :keep_alive_timeout, :integer,  :default => 300
 
     def configure(conf)
         super
@@ -47,7 +53,9 @@ module Fluent
             credentials['user']     = @user
             credentials['password'] = @password
         end
-        @wendelin = WendelinClient.new(@streamtool_uri, credentials, @log)
+        @wendelin = WendelinClient.new(@streamtool_uri, credentials, @log,
+                                       @ssl_timeout, @open_timeout,
+                                       @read_timeout, @keep_alive_timeout)
     end
 
     def start
@@ -80,7 +88,11 @@ module Fluent
         # further on server by Wendelin
         reference = tag
 
-        @wendelin.ingest(reference, data_chunk)
+        if @use_keep_alive
+          @wendelin.ingest_with_keep_alive(reference, data_chunk)
+        else
+          @wendelin.ingest(reference, data_chunk)
+        end
     end
 
   end

data/lib/fluent/plugin/wendelin_client.rb

@@ -15,91 +15,174 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-
 require 'net/http'
 require 'openssl'
 
-
 # class representing a Wendelin client
 class WendelinClient
 
   # `streamtool_uri` - URI pointing to portal_input_data_stream "mountpoint"
   # `credentials`    # {'user' => _, 'password' => _}     TODO change to certificate
   # `log` - logger to use
-  def initialize(streamtool_uri, credentials, log)
-    @streamtool_uri = streamtool_uri
-    @credentials    = credentials
-    @log            = log
+  def initialize(streamtool_uri, credentials, log,
+                 ssl_timeout, open_timeout, read_timeout, keep_alive_timeout)
+    @streamtool_uri     = streamtool_uri
+    @credentials        = credentials
+    @log                = log
+    @ssl_timeout        = ssl_timeout
+    @open_timeout       = open_timeout
+    @read_timeout       = read_timeout
+    @keep_alive_timeout = keep_alive_timeout
   end
 
 
+  # start request in an independent function to keep the connection open
+  def start_connection(uri)
+
+    @log.debug "start new connection"
+
+    @http = Net::HTTP.start(uri.hostname, uri.port,
+      :use_ssl => (uri.scheme == 'https'),
+      :verify_mode  => OpenSSL::SSL::VERIFY_NONE,
+
+      # Net::HTTP default open timeout is infinity, which results
+      # in thread hang forever if other side does not fully
+      # establish connection. Default read_timeout is 60 seconds.
+      # We go safe way and make sure all timeouts are defined.
+      :ssl_timeout  => @ssl_timeout,
+      :open_timeout => @open_timeout,
+      :read_timeout => @read_timeout,
+      :keep_alive_timeout => @keep_alive_timeout,)
+  end
+
   # ingest `data_chunk` to a stream referenced as `reference`
-  def ingest(reference, data_chunk)
-      uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
-      req = Net::HTTP::Post.new(uri)
-      if @credentials.has_key?('user')
-        req.basic_auth @credentials['user'], @credentials['password']
-      end
-
-      # When using 'application/x-www-form-urlencoded', Ruby encodes with regex
-      # and it is far too slow. Such POST is legit:
-      # https://stackoverflow.com/a/14710450
-      req.body = data_chunk
-      req.content_type = 'application/octet-stream'
-
-      @log.on_trace do
-          @log.trace '>>> REQUEST'
-          @log.trace "method\t=> #{req.method}"
-          @log.trace "path\t=> #{req.path}"
-          @log.trace "uri\t=> #{req.uri}"
-          @log.trace "body\t=> #{req.body}"
-          @log.trace "body_stream\t=> #{req.body_stream}"
-          req.each {|h| @log.trace "#{h}:\t#{req[h]}"}
-          @log.trace
-      end
-
-      begin
-          # TODO keep connection open (so that every new ingest does not do
-          # full connect again)
-          res = Net::HTTP.start(uri.hostname, uri.port,
-                      :use_ssl      => (uri.scheme == 'https'),
-                      # NOTE = "do not check server cert"
-                      # TODO move this out to conf parameters
-                      :verify_mode  => OpenSSL::SSL::VERIFY_NONE,
-
-                      # Net::HTTP default open timeout is infinity, which results
-                      # in thread hang forever if other side does not fully
-                      # establish connection. Default read_timeout is 60 seconds.
-                      # We go safe way and make sure all timeouts are defined.
-                      :ssl_timeout  => 60,
-                      :open_timeout => 60,
-                      :read_timeout => 60,
-                ) do |http|
-              http.request(req)
-          end
-
-      rescue
-          # some http/ssl/other connection error
-          @log.warn "HTTP ERROR:"
-          raise
-
-      else
-          @log.on_trace do
-              @log.trace '>>> RESPONSE'
-              res.each {|h| @log.trace "#{h}:\t#{res[h]}"}
-              @log.trace "code\t=> #{res.code}"
-              @log.trace "msg\t=> #{res.message}"
-              @log.trace "class\t=> #{res.class}"
-              @log.trace "body:", res.body
-          end
-
-          if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
-              #@log.info "ingested ok"
-          else
-              @log.warn "FAIL:"
-              res.value
-          end
-      end
+  def ingest_with_keep_alive(reference, data_chunk)
+    uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
+    # call start_connection if http is undefined
+    if ! defined? @http
+      start_connection(uri)
+    end
+
+    # connect again if the connection is not started
+    if ! @http.started?()
+      start_connection(uri)
+    end
+
+    @request = Net::HTTP::Post.new(uri)
+
+    # When using 'application/x-www-form-urlencoded', Ruby encodes with regex
+    # and it is far too slow. Such POST is legit:
+    # https://stackoverflow.com/a/14710450
+    @request.body = data_chunk
+    @request.content_type = 'application/octet-stream'
+
+    if @credentials.has_key?('user')
+      @request.basic_auth @credentials['user'], @credentials['password']
+    end
+
+    @log.on_trace do
+      @log.trace '>>> REQUEST'
+      @log.trace "method\t=> #{@request.method}"
+      @log.trace "path\t=> #{@request.path}"
+      @log.trace "uri\t=> #{@request.uri}"
+      @log.trace "body\t=> #{@request.body}"
+      @log.trace "body_stream\t=> #{@request.body_stream}"
+      @request.each {|h| @log.trace "#{h}:\t#{@request[h]}"}
+      @log.trace
+    end
+
+    begin
+      res = @http.request(@request) # Net::HTTPResponse object
+    end
+
+    rescue
+      # some http/ssl/other connection error
+      @log.warn "HTTP ERROR:"
+    raise
+  else
+    @log.on_trace do
+      @log.trace '>>> RESPONSE'
+      res.each {|h| @log.trace "#{h}:\t#{res[h]}"}
+      @log.trace "code\t=> #{res.code}"
+      @log.trace "msg\t=> #{res.message}"
+      @log.trace "class\t=> #{res.class}"
+      @log.trace "body:", res.body
+    end
+
+    if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
+      #@log.info "ingested ok"
+    else
+      @log.warn "FAIL:"
+      res.value
+    end
   end
 
+  def ingest(reference, data_chunk)
+     uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
+     req = Net::HTTP::Post.new(uri)
+     if @credentials.has_key?('user')
+       req.basic_auth @credentials['user'], @credentials['password']
+     end
+
+     # When using 'application/x-www-form-urlencoded', Ruby encodes with regex
+     # and it is far too slow. Such POST is legit:
+     # https://stackoverflow.com/a/14710450
+     req.body = data_chunk
+     req.content_type = 'application/octet-stream'
+
+     @log.on_trace do
+         @log.trace '>>> REQUEST'
+         @log.trace "method\t=> #{req.method}"
+         @log.trace "path\t=> #{req.path}"
+         @log.trace "uri\t=> #{req.uri}"
+         @log.trace "body\t=> #{req.body}"
+         @log.trace "body_stream\t=> #{req.body_stream}"
+         req.each {|h| @log.trace "#{h}:\t#{req[h]}"}
+         @log.trace
+     end
+
+     begin
+         # TODO keep connection open (so that every new ingest does not do
+         # full connect again)
+         res = Net::HTTP.start(uri.hostname, uri.port,
+                     :use_ssl      => (uri.scheme == 'https'),
+                     # NOTE = "do not check server cert"
+                     # TODO move this out to conf parameters
+                     :verify_mode  => OpenSSL::SSL::VERIFY_NONE,
+
+                     # Net::HTTP default open timeout is infinity, which results
+                     # in thread hang forever if other side does not fully
+                     # establish connection. Default read_timeout is 60 seconds.
+                     # We go safe way and make sure all timeouts are defined.
+                     :ssl_timeout  => @ssl_timeout,
+                     :open_timeout => @open_timeout,
+                     :read_timeout => @read_timeout,
+               ) do |http|
+             http.request(req)
+         end
+
+     rescue
+         # some http/ssl/other connection error
+         @log.warn "HTTP ERROR:"
+         raise
+
+     else
+         @log.on_trace do
+             @log.trace '>>> RESPONSE'
+             res.each {|h| @log.trace "#{h}:\t#{res[h]}"}
+             @log.trace "code\t=> #{res.code}"
+             @log.trace "msg\t=> #{res.message}"
+             @log.trace "class\t=> #{res.class}"
+             @log.trace "body:", res.body
+         end
+
+         if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
+             #@log.info "ingested ok"
+         else
+             @log.warn "FAIL:"
+             res.value
+         end
+     end
+ end
+
 end

metadata

@@ -1,37 +1,40 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-wendelin
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  prerelease: 
+  version: '0.4'
 platform: ruby
 authors:
-- Kirill Smelkov
+- Nexedians
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-18 00:00:00.000000000 Z
+date: 2018-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  type: :runtime
   name: fluentd
-  requirement: !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
 description: Fluentd output plugin to forward data to Wendelin system
 email:
-- kirr@nexedi.com
+- klaus@nexedi.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -45,25 +48,26 @@ files:
 homepage: https://lab.nexedi.com/nexedi/fluent-plugin-wendelin
 licenses:
 - Apache-2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.1
+rubygems_version: 1.8.23.2
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Fluentd output to Wendelin
 test_files: []

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 16421dddd4db913fcdc9c1c50de6c83065014e9c
-  data.tar.gz: 7af5466ec21e0b9845ff1fef6e1e85d2b4fd5a56
-SHA512:
-  metadata.gz: 7ecff204b3f1b6ccc090709ecf00f88fa511be03e9912e1a1aa5761ba9e0a39b871df297e50796a26c9640f266eeed198d061dc502079677053e3dac107b12e7
-  data.tar.gz: 98be162b72588df472140aa7c2eb033c70c3ac4d36fc9fde4c4b3d6ab85934bcb5e9c96320bf781c0f41e351494f0818cd60f0e7041950157558b67838dc30da