logstash-filter-zeromq 2.0.4 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9a571be2bf820b9a4e19404f9a4eb6ecec99a6b
-  data.tar.gz: afed6933e2b649926e0bb4732114c0d08f463a38
+  metadata.gz: 48c16132cbb65dec76d2a5ebf21265278767b372
+  data.tar.gz: af27398f55f31de69829946bd4c1d52312213631
 SHA512:
-  metadata.gz: 4a3b8df1d7b6ff7fbe91a48236caf35db756909ddee44d9595b08d8ca2b34e0e362d1543c9fba10e886f58bd75bc9f34c193c227d6592be7011b22bc6cd99b4a
-  data.tar.gz: 2247e68c40c1f6fe4576732c406577765485df3929db3f07a93d3f65cc4f57a58df588d0354af922a13b491928f9c9fba68fec3646b34edd00103981fdb9194c
+  metadata.gz: 9c2e51cab401a165b44ce818a3a4ee1a768ebe863f53f0a61c3dbbd4ed388e06b1ef45b8ddfdb8a48bedf1cea122fcd2d074225987a8dd880809699b9d23c8fb
+  data.tar.gz: f42425f4cfca6f31c4d5e1f7d3dafef3bd4464a4b9e2dfcae818b8b7b3a54716a82b6df460818d70aebd16d246e9ba3e02941a23feec3cd6dd246e652efe753d

data/CHANGELOG.md

@@ -1,9 +1,14 @@
-# 2.0.4
+## 2.1.1
+ - make sure we correctly encode the object #7
+## 2.1.0
+  - Use logstash-mixin-zeromq
+  - Add tests
+  - Use a sentinel value to signal event cancellation instead of an empty string
+## 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.3
+## 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
-

data/CONTRIBUTORS

@@ -10,6 +10,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Tom Howe (turtlebender)
+* Avishai Ish-Shalom (avishai-ish-shalom)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-zeromq-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-zeromq-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-zeromq.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-zeromq)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/filters/zeromq.rb

@@ -29,6 +29,10 @@ class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
   # If this is unset, the whole event will be sent
   config :field, :validate => :string
 
+  # A sentinel value to signal the filter to cancel the event
+  # If the peer returns the sentinel value, the event will be cancelled
+  config :sentinel, :validate => :string, :default => ""
+
   # 0mq mode
   # server mode binds/listens
   # client mode connects
@@ -71,8 +75,8 @@ class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
   public
   def register
     require "ffi-rzmq"
-    require "logstash/util/zeromq"
-    self.class.send(:include, LogStash::Util::ZeroMQ)
+    require "logstash/plugin_mixins/zeromq"
+    self.class.send(:include, LogStash::PluginMixins::ZeroMQ)
     connect
   end #def register
 
@@ -121,13 +125,13 @@ class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
   def send_recv(message)
     success = false
     @retries.times do
-      @logger.debug("0mq: sending", :request => message)
+      @logger.debug? && @logger.debug("0mq: sending", :request => message)
       rc = @zsocket.send_string(message)
       if ZMQ::Util.resultcode_ok?(rc)
         success = true
         break
       else
-        @logger.debug("0mq: error sending message (zmq_errno = #{ZMQ::Util.errno}, zmq_error_string = '#{ZMQ::Util.error_string}'")
+        @logger.debug? && @logger.debug("0mq: error sending message (zmq_errno = #{ZMQ::Util.errno}, zmq_error_string = '#{ZMQ::Util.error_string}'")
         reconnect
       end #if resultcode
     end #retries.times
@@ -142,13 +146,13 @@ class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
     reply = ''
     success = false
     @retries.times do
-      @logger.debug("0mq: polling for reply for #{@timeout}ms.")
+      @logger.debug? && @logger.debug("0mq: polling for reply for #{@timeout}ms.")
       #poll the socket. If > 0, something to read. If < 0, error. If zero, loop
       num_readable = @poller.poll(@timeout)
       if num_readable > 0
         #something to read, do it.
         rc = @zsocket.recv_string(reply)
-        @logger.debug("0mq: message received, checking error")
+        @logger.debug? && @logger.debug("0mq: message received, checking error")
         error_check(rc, "in recv_string")
         success = true
         break
@@ -171,8 +175,6 @@ class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
 
   public
   def filter(event)
-    
-
     begin
       # TODO (lusis): Allow filtering multiple fields
       if @field
@@ -180,19 +182,19 @@ class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
       else
         success, reply = send_recv(event.to_json)
       end
-      # If we receive an empty reply, this is an indication that the filter
+      # If we receive an sentinel value as reply, this is an indication that the filter
       # wishes to cancel this event.
-      if reply.empty?
-        @logger.debug("0mq: recieved empty reply, cancelling event.")
+      if success && reply == @sentinel
+        @logger.debug? && @logger.debug("0mq: recieved sentinel value, cancelling event.")
         event.cancel
         return
       end
-      @logger.debug("0mq: receiving", :reply => reply)
+      @logger.debug? && @logger.debug("0mq: receiving", :reply => reply)
       if @field
         event[@field] = event.sprintf(reply)
         filter_matched(event)
       else
-        reply = LogStash::Json.load(reply)
+        reply = LogStash::Event.new(LogStash::Json.load(reply))
         event.overwrite(reply)
       end
       filter_matched(event)

data/logstash-filter-zeromq.gemspec

@@ -1,10 +1,11 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-zeromq'
-  s.version         = '2.0.4'
+
+  s.version         = '2.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "ZeroMQ filter. This is a way to send an event externally for filtering"
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -22,8 +23,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
 
-  s.add_runtime_dependency 'ffi-rzmq', '~> 2.0.4'
+  s.add_runtime_dependency "logstash-mixin-zeromq", ">= 2.0.1"
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/filters/zeromq_spec.rb

@@ -1,6 +1,56 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/filters/zeromq'
+require 'json'
+require 'securerandom'
 
 describe LogStash::Filters::ZeroMQ do
+  let(:sentinel) { SecureRandom.hex[1..2] }
+  let(:filter) { LogStash::Plugin.lookup("filter", "zeromq").new({"sentinel" => sentinel}) }
+  let(:event) do
+    LogStash::Event.new({
+      "message" => "some message",
+      "seq" => rand(1000),
+      "field1" => "some field value",
+      "@version" => "1",
+      "@timestamp" => Time.now.utc.iso8601(3)
+    })
+  end
 
+  let(:event_text) { event.to_json }
+
+  before do
+    allow(filter).to receive(:connect)
+  end
+
+  it "should send entire event as json if field not specified" do
+    expect(filter).to receive(:send_recv).with(event_text)
+    filter.filter(event)
+  end
+
+  it "should cancel event if peer returns an answer==sentinel" do
+    expect(filter).to receive(:send_recv).and_return([true, sentinel])
+    expect(event).to receive(:cancel)
+    filter.filter(event)
+  end
+
+  it "should not cancel event if send_recv failed" do
+    expect(filter).to receive(:send_recv).and_return([false, event_text])
+    expect(event).to_not receive(:cancel)
+    filter.filter(event)
+  end
+
+  context "Filter specific field" do
+    let(:filter) { LogStash::Plugin.lookup("filter", "zeromq").new({"field" => "field1"}) }
+
+    it "should send only a single field" do
+      expect(filter).to receive(:send_recv).with(event["field1"])
+      filter.filter(event)
+    end
+
+    it "should not cancel event if send_recv failed and event field==sentinel" do
+      expect(filter).to receive(:send_recv).and_return([false, sentinel])
+      expect(event).to_not receive(:cancel)
+      filter.filter(event)
+    end
+  end
 end

metadata

@@ -1,58 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-zeromq
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-zeromq
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
+        version: 2.0.1
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.4
-  name: ffi-rzmq
+        version: 2.0.1
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.4
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -65,7 +65,6 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/filters/zeromq.rb
-- lib/logstash/util/zeromq.rb
 - logstash-filter-zeromq.gemspec
 - spec/filters/zeromq_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -80,12 +79,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []

data/lib/logstash/util/zeromq.rb

@@ -1,47 +0,0 @@
-# encoding: utf-8
-require 'ffi-rzmq'
-require "logstash/namespace"
-
-module LogStash::Util::ZeroMQ
-  CONTEXT = ZMQ::Context.new
-  # LOGSTASH-400
-  # see https://github.com/chuckremes/ffi-rzmq/blob/master/lib/ffi-rzmq/socket.rb#L93-117
-  STRING_OPTS = %w{IDENTITY SUBSCRIBE UNSUBSCRIBE}
-
-  def context
-    CONTEXT
-  end
-
-  def setup(socket, address)
-    if server?
-      error_check(socket.bind(address), "binding to #{address}")
-    else
-      error_check(socket.connect(address), "connecting to #{address}")
-    end
-    @logger.info("0mq: #{server? ? 'connected' : 'bound'}", :address => address)
-  end
-
-  def error_check(rc, doing)
-    unless ZMQ::Util.resultcode_ok?(rc)
-      @logger.error("ZeroMQ error while #{doing}", { :error_code => rc })
-      raise "ZeroMQ Error while #{doing}"
-    end
-  end # def error_check
-
-  def setopts(socket, options)
-    options.each do |opt,value|
-      sockopt = opt.split('::')[1]
-      option = ZMQ.const_defined?(sockopt) ? ZMQ.const_get(sockopt) : ZMQ.const_missing(sockopt)
-      unless STRING_OPTS.include?(sockopt)
-        begin
-          Float(value)
-          value = value.to_i
-        rescue ArgumentError
-          raise "#{sockopt} requires a numeric value. #{value} is not numeric"
-        end
-      end # end unless
-      error_check(socket.setsockopt(option, value),
-              "while setting #{opt} == #{value}")
-    end # end each
-  end # end setopts
-end # module LogStash::Util::ZeroMQ