logstash-filter-zeromq 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e3dcdef0b8b0bcd244c53f52988a55af838825ef
+  data.tar.gz: 0d5bf5f6fa3781c2a0a9d7a82dd312d882e4cf59
+SHA512:
+  metadata.gz: 936eff3ec8d09106d1186e69a5efc0f689b221961eebb18dc2b4f2d8b03e595da41314ca4c9c7cc19d2384b97400426e702528c5ab6bf009e9f54f158f03c7d9
+  data.tar.gz: d53c4b023a10ac06d8726d0a6f9171a503de0fe74a538550f8543374eeddc3fec2397eb324752a58290a956e1ba617d3b317238287b163b4c82cc0953022cafa

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/filters/zeromq.rb

@@ -0,0 +1,214 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# ZeroMQ filter. This is the best way to send an event externally for filtering
+# It works much like an exec filter would by sending the event "offsite"
+# for processing and waiting for a response
+#
+# The protocol here is:
+#   * REQ sent with JSON-serialized logstash event
+#   * REP read expected to be the full JSON 'filtered' event
+#   * - if reply read is an empty string, it will cancel the event.
+#
+# Note that this is a limited subset of the zeromq functionality in
+# inputs and outputs. The only topology that makes sense here is:
+# REQ/REP. 
+class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
+
+  config_name "zeromq"
+  milestone 1
+
+  # 0mq socket address to connect or bind
+  # Please note that inproc:// will not work with logstash
+  # as we use a context per thread
+  # By default, filters connect
+  config :address, :validate => :string, :default => "tcp://127.0.0.1:2121"
+
+  # The field to send off-site for processing
+  # If this is unset, the whole event will be sent
+  # TODO (lusis)
+  # Allow filtering multiple fields
+  config :field, :validate => :string
+
+  # 0mq mode
+  # server mode binds/listens
+  # client mode connects
+  config :mode, :validate => ["server", "client"], :default => "client"
+
+  # timeout in milliseconds on which to wait for a reply.
+  config :timeout, :validate => :number, :default => 500
+  
+  # number of retries, used for both sending and receiving messages.
+  # for sending, retries should return instantly.
+  # for receiving, the total blocking time is up to retries X timeout, 
+  # which by default is 3 X 500 = 1500ms
+  config :retries, :validate => :number, :default => 3
+
+  # tag to add if zeromq timeout expires before getting back an answer.
+  # If set to "" then no tag will be added.
+  config :add_tag_on_timeout, :validate => :string, :default => "zeromqtimeout"
+
+  # 0mq socket options
+  # This exposes zmq_setsockopt
+  # for advanced tuning
+  # see http://api.zeromq.org/2-1:zmq-setsockopt for details
+  #
+  # This is where you would set values like:
+  # ZMQ::HWM - high water mark
+  # ZMQ::IDENTITY - named queues
+  # ZMQ::SWAP_SIZE - space for disk overflow
+  # ZMQ::SUBSCRIBE - topic filters for pubsub
+  #
+  # example: sockopt => ["ZMQ::HWM", 50, "ZMQ::IDENTITY", "my_named_queue"]
+  config :sockopt, :validate => :hash
+
+  public
+  def initialize(params)
+    super(params)
+
+    @threadsafe = false
+  end
+
+  public
+  def register
+    require "ffi-rzmq"
+    require "logstash/util/zeromq"
+    self.class.send(:include, LogStash::Util::ZeroMQ)
+    connect
+  end #def register
+
+  public
+  def teardown
+    close
+    super()
+  end #def teardown
+
+  private
+  def close
+    @logger.debug("0mq: closing socket.")
+    @poller.deregister(@zsocket, ZMQ::POLLIN)
+    @zsocket.close
+  end #def close
+
+  private
+  def connect
+    @logger.debug("0mq: connecting socket")
+    @zsocket = context.socket(ZMQ::REQ)
+    error_check(@zsocket.setsockopt(ZMQ::LINGER, 0),
+                "while setting ZMQ::LINGER == 0)")
+    @poller = ZMQ::Poller.new
+    @poller.register(@zsocket, ZMQ::POLLIN)
+
+    if @sockopt
+      #TODO: should make sure that ZMQ::LINGER and ZMQ::POLLIN are not changed
+      setopts(@zsocket, @sockopt)
+    end
+
+    setup(@zsocket, @address)
+  end #def connect
+
+  private
+  def reconnect
+    close
+    connect
+  end #def reconnect
+
+  #send and receive data. message is assumed to be json
+  #will return a boolean for success, and a string containing one of several things:
+  #  - empty string: response from server
+  #  - updated string: response from server
+  #  - original message: could not send request or get response from server in time 
+  private
+  def send_recv(message)
+    success = false
+    @retries.times do
+      @logger.debug("0mq: sending", :request => message)
+      rc = @zsocket.send_string(message) 
+      if ZMQ::Util.resultcode_ok?(rc)
+        success = true
+        break
+      else
+        @logger.debug("0mq: error sending message (zmq_errno = #{ZMQ::Util.errno}, zmq_error_string = '#{ZMQ::Util.error_string}'")
+        reconnect
+      end #if resultcode
+    end #retries.times
+
+    #if we did not succeed log it and fail here.
+    if not success
+      @logger.warn("0mq: error sending message (zmq_errno = #{ZMQ::Util.errno}, zmq_error_string = '#{ZMQ::Util.error_string}'")
+      return success, message 
+    end
+
+    #now get reply
+    reply = ''
+    success = false
+    @retries.times do 
+      @logger.debug("0mq: polling for reply for #{@timeout}ms.")
+      #poll the socket. If > 0, something to read. If < 0, error. If zero, loop
+      num_readable = @poller.poll(@timeout)
+      if num_readable > 0
+        #something to read, do it.
+        rc = @zsocket.recv_string(reply)
+        @logger.debug("0mq: message received, checking error")
+        error_check(rc, "in recv_string")
+        success = true
+        break
+      elsif num_readable < 0
+        #error, reconnect
+        close
+        connect
+      end
+    end # @retries.times
+     
+    #if we maxed out on number of retries, then set reply to message so that
+    #the event isn't cancelled. we want to carry on if the server is down.
+    if not success 
+      @logger.warn("0mq: did not receive reply (zmq_errno = #{ZMQ::Util.errno}, zmq_error_string = '#{ZMQ::Util.error_string}'")
+      return success, message 
+    end
+
+    return success, reply
+  end #def send_recv
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    begin
+      if @field
+      	success, reply = send_recv(event[@field])
+      else
+        success, reply = send_recv(event.to_json)
+      end
+      # If we receive an empty reply, this is an indication that the filter
+      # wishes to cancel this event.
+      if reply.empty?
+        @logger.debug("0mq: recieved empty reply, cancelling event.")
+        event.cancel
+        return
+      end
+      @logger.debug("0mq: receiving", :reply => reply)
+      if @field
+        event[@field] = event.sprintf(reply)
+        filter_matched(event)
+      else
+        reply = JSON.parse(reply)
+        event.overwrite(reply)
+      end
+      filter_matched(event)
+      #if message send/recv was not successful add the timeout
+      if not success
+        (event["tags"] ||= []) << @add_tag_on_timeout
+      end
+    rescue => e
+      @logger.warn("0mq filter exception", :address => @address, :exception => e, :backtrace => e.backtrace)
+    end
+  end # def filter
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+end # class LogStash::Filters::ZeroMQ

data/logstash-filter-zeromq.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-zeromq'
+  s.version         = '0.1.2'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "ZeroMQ filter. This is a way to send an event externally for filtering"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  s.add_runtime_dependency 'ffi-rzmq', ['1.0.0']
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/filters/zeromq_spec.rb

@@ -0,0 +1,6 @@
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/filters/zeromq'
+
+describe LogStash::Filters::ZeroMQ do
+
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-zeromq
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: ffi-rzmq
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- lib/logstash/filters/zeromq.rb
+- logstash-filter-zeromq.gemspec
+- spec/filters/zeromq_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.4
+signing_key:
+specification_version: 4
+summary: ZeroMQ filter. This is a way to send an event externally for filtering
+test_files:
+- spec/filters/zeromq_spec.rb