logstash-output-faye 2.2.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c84a8228af88478051f9d33450c58d8cb331f7a9
-  data.tar.gz: 5321b905941a648b842dcf3267a70a4164f08c15
+  metadata.gz: e9c4c6132f6005917ee6a4849af20dfd63237066
+  data.tar.gz: 34c5394325a6af71ed5b1bb28e8be2e9146a7933
 SHA512:
-  metadata.gz: a9a960a36ac96bcb68e8bab72d27d91583853a2806114f4508548bc20362e2b3c2ffde554e58fd51e2c1ff4754d3a8f81256a1e2bede631252ceb6f34b641692
-  data.tar.gz: ebd9f5caaf0114f064da65cfc91d40cc6e86ed6dc3368faac1b200aac876c3f5bc43c60fc5818d34115393d55a230d9c8023d9879c36f99624fd4e3bbb47f2ef
+  metadata.gz: 5e75162f94bbb426e00b115531b963e41000b5ffbd2de5c07f26fe599787bca8d080c8c0484cf5980bb744fb9a8b5b7a2f7ca3eb9d30f2fe8af3d3dd6df80939
+  data.tar.gz: 9b5a9024fa5ceb6fa1ab3ed75b5a2c87e7c25a298011e09c78384dd04fcd3b6211f7669413fbfb215bf3cb43a1b1839694632dd0eb4679ceccc46b49dc325e44

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.0
+ - New strukture from current http-plugin
+
 ## 2.2.0
  - Changed dependency to new logstash-core-plugin-api
 

data/lib/logstash/outputs/faye.rb

@@ -1,15 +1,23 @@
 # encoding: utf-8
-
-require "logstash/namespace"
 require "logstash/outputs/base"
-require 'json'
-require 'pry'
+require "logstash/namespace"
+require "logstash/json"
+require "uri"
+require "logstash/plugin_mixins/http_client"
 
-# File output.
-#
-# Write events to files on disk. You can use fields from the
-# event as parts of the filename.
 class LogStash::Outputs::Faye < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+
+  # This output lets you send events to a
+  # generic HTTP(S) endpoint
+  #
+  # This output will execute up to 'pool_max' requests in parallel for performance.
+  # Consider this when tuning this plugin for performance.
+  #
+  # Additionally, note that when parallel execution is used strict ordering of events is not
+  # guaranteed!
+  #
+  # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
 
   config_name "faye"
 
@@ -19,90 +27,139 @@ class LogStash::Outputs::Faye < LogStash::Outputs::Base
   # Faye-security-token
   config :faye_token, :validate => :string
 
-  # Url of the faye-server
+  # URL to use
   config :url, :validate => :string, :default => "http://localhost:9292/faye"
 
-  # If TLS/SSL is used (because the `url` is "https://...", then this
-  # setting will determine if certificate validation is done.
-  #
-  # Note: If you set this to false, you will be destroying the security
-  # features provided by TLS. Setting this to false is never recommended,
-  # especially never in production.
-  config :verify_ssl, :validate => :boolean, :default => true
-
   # This lets you choose the structure and parts of the event that are sent.
   #
-  # For example:
   #
-  #    mapping => ["foo", "%{host}", "bar", "%{type}"]
+  # For example:
+  # [source,ruby]
+  #    mapping => {"foo" => "%{host}" 
+  #               "bar" => "%{type}"}
   config :mapping, :validate => :hash
 
-  public
   def register
-    require "net/http"
-  end # def register
+    @http_method = :post
+    @content_type = "application/json"
 
-  public
-  def receive(event)
-    return unless output?(event)
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
 
-  _send_message event
-  end # def receive
+    @requests = Array.new
+  end # def register
 
-  private
-  def _send_message(event)
+  def multi_receive(events)
+    events.each {|event| receive(event, :parallel)}
+    client.execute!
+  end
 
-    if @mapping
-        message = Hash.new
-        @mapping.each do |k,v|
-            message[k] = event.sprintf(v)
-        end
-    else
-        message = event.to_hash
-    end
+  # Once we no longer need to support Logstash < 2.2 (pre-ng-pipeline)
+  # We don't need to handle :background style requests
+  #
+  # We use :background style requests for Logstash < 2.2 because before the microbatching
+  # pipeline performance is greatly improved by having some degree of async behavior.
+  #
+  # In Logstash 2.2 and after things are much simpler, we just run each batch in parallel
+  # This will make performance much easier to reason about, and more importantly let us guarantee
+  # that if `multi_receive` returns all items have been sent.
+  def receive(event, async_type=:background)
 
     if @faye_token
-      message = {:channel => @channel, :data => message, :ext => {:auth_token => @faye_token}}
+      body = LogStash::Json.dump({:channel => @channel, :data => map_event(event), :ext => {:auth_token => @faye_token}})
     else
-      message = {:channel => @channel, :data => message}
+      body = LogStash::Json.dump({:channel => @channel, :data => map_event(event)})
     end
 
-    uri = URI.parse(@url)
+    # Block waiting for a token
+    token = @request_tokens.pop if async_type == :background
+
+    # Send the request
+    url = event.sprintf(@url)
+    headers = event_headers(event)
 
-    begin
-      http = Net::HTTP.new(uri.host, uri.port)
+    # Create an async request
+    request = client.send(async_type).send(@http_method, url, :body => body, :headers => headers)
 
-      if uri.scheme == 'https'
-        http.use_ssl = true
+    request.on_complete do
+      # Make sure we return the token to the pool
+      @request_tokens << token  if async_type == :background
+    end
 
-        if !@verify_ssl
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        else
-          http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        end
+    request.on_success do |response|
+      if response.code < 200 || response.code > 299
+        log_failure(
+          "Encountered non-200 HTTP code #{response.code}",
+          :response_code => response.code,
+          :url => url,
+          :event => event.to_hash)
       end
+    end
+
+    request.on_failure do |exception|
+      log_failure("Could not fetch URL",
+                  :url => url,
+                  :method => @http_method,
+                  :body => faye_body.to_json,
+                  :headers => headers,
+                  :message => exception.message,
+                  :class => exception.class.name,
+                  :backtrace => exception.backtrace
+      )
+    end
+
+    request.call if async_type == :background
+  end
+
+  def close
+    client.close
+  end
 
-      request = Net::HTTP::Post.new(uri.path)
-      request.set_form_data(:message => message.to_json)
+  private
 
-      response = http.request(request)
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
 
-      if response.is_a?(Net::HTTPSuccess)
-        message = JSON.parse(response.body)
+  # Manticore doesn't provide a way to attach handlers to background or async requests well
+  # It wants you to use futures. The #async method kinda works but expects single thread batches
+  # and background only returns futures.
+  # Proposed fix to manticore here: https://github.com/cheald/manticore/issues/32
+  def request_async_background(request)
+    @method ||= client.executor.java_method(:submit, [java.util.concurrent.Callable.java_class])
+    @method.call(request)
+  end
 
-        @logger.warn("Faye request was not successfully", message.first) unless message.first['successful']
-      else
-        @logger.warn("Faye response not ok", :http_status => response.code, :message => response.message)
+  def map_event(event)
+    if @mapping
+      @mapping.reduce({}) do |acc,kv|
+        k,v = kv
+        acc[k] = event.sprintf(v)
+        acc
       end
-    rescue Exception => e
-      @logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
+    else
+      event.to_hash
     end
-
   end
 
-  def teardown
-    finished
+  def event_headers(event)
+    headers = custom_headers(event) || {}
+    headers["Content-Type"] = @content_type
+    headers
   end
-end # class LogStash::Outputs::Faye
 
+  def custom_headers(event)
+    return nil unless @headers
 
+    @headers.reduce({}) do |acc,kv|
+      k,v = kv
+      acc[k] = event.sprintf(v)
+      acc
+    end
+  end
+end

data/logstash-output-faye.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-faye'
-  s.version         = '2.2.0'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Send event to faye-message-server"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,10 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.20", "<= 2.99"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 2.2.1", "< 5.0.0"
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-faye
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Signify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-10 00:00:00.000000000 Z
+date: 2016-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,26 @@ dependencies:
     - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+    - - <
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  name: logstash-mixin-http_client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+    - - <
+      - !ruby/object:Gem::Version
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -44,6 +64,34 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sinatra
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: dietmar@signifydata.com
 executables: []