logstash-output-zeromq 2.0.4 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71feb458cc271212d00f8269ca2891dfca2d64d2
-  data.tar.gz: 5af956b08a2e26c7cbdbf0c60fe5c71c543afee0
+  metadata.gz: f1408116f878e88c8710d7c0d2a9f3aed0da4995
+  data.tar.gz: 019a6cf56a05c3b58cabee42ca7dc2077f86f4c2
 SHA512:
-  metadata.gz: b77a42adddf951abc6adf74cdc36d5cd98ed703d8715c79f043c3552e562dcfa12ba3101f49448deb8bffa14a639a4a50cd54848cf206cecffb38207b3794ebd
-  data.tar.gz: f735b020d4ed6a225a57b4181d2b57a9bdccd37016ce5c49ab1261952278e8753c0827f4fafaf34169a33d7e5007504ca9dcc97a2f79e784706da20aa6bb5bcb
+  metadata.gz: 69e182757ddd7af8eca29da2e23b09d5438086917f0e3b8863c518d2c1991352c5d2fc13c053e8024d90387d4021eafd0c859fdbe3eafa4ce89cb8fd1ac9337f
+  data.tar.gz: 8faff7f09233a20b0b17a7b4d7b70db0e511a27d452431a61f6d4ef18961fef2fb0767ac2ed9556f67cea9881b7c1045cc97af4382e9541cd9fec47f66cb2296

data/CHANGELOG.md

@@ -1,9 +1,11 @@
+# 2.1.0
+  - Depend on logstash-mixin-zeromq, remove lib/logstash/util/zeromq
+  - Use new plugin API
 # 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
-

data/CONTRIBUTORS

@@ -10,6 +10,7 @@ Contributors:
 * Nick Padilla (NickPadilla)
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
+* Avishai Ish-Shalom (avishai-ish-shalom)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-zeromq-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-zeromq-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-zeromq.svg)](https://travis-ci.org/logstash-plugins/logstash-output-zeromq)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/outputs/zeromq.rb

@@ -26,7 +26,7 @@ class LogStash::Outputs::ZeroMQ < LogStash::Outputs::Base
   #
   # * pushpull - inputs are pull, outputs are push
   # * pubsub - inputs are subscribers, outputs are publishers
-  # * pair - inputs are clients, inputs are servers
+  # * pair - inputs are clients, outputs are servers
   #
   # If the predefined topology flows don't work for you,
   # you can change the 'mode' setting
@@ -61,36 +61,13 @@ class LogStash::Outputs::ZeroMQ < LogStash::Outputs::Base
 
   public
   def register
-    require "ffi-rzmq"
-    require "logstash/util/zeromq"
-    self.class.send(:include, LogStash::Util::ZeroMQ)
+    load_zmq
 
     if @mode == "server"
       workers_not_supported("With 'mode => server', only one zeromq socket may bind to a port and may not be shared among threads. Going to single-worker mode for this plugin!")
     end
 
-    # Translate topology shorthand to socket types
-    case @topology
-    when "pair"
-      zmq_const = ZMQ::PAIR
-    when "pushpull"
-      zmq_const = ZMQ::PUSH
-    when "pubsub"
-      zmq_const = ZMQ::PUB
-    end # case socket_type
-
-    @zsocket = context.socket(zmq_const)
-
-    error_check(@zsocket.setsockopt(ZMQ::LINGER, 1),
-                "while setting ZMQ::LINGER == 1)")
-
-    if @sockopt
-      setopts(@zsocket, @sockopt)
-    end
-
-    @address.each do |addr|
-      setup(@zsocket, addr)
-    end
+    connect
 
     @codec.on_event(&method(:publish))
   end # def register
@@ -104,28 +81,55 @@ class LogStash::Outputs::ZeroMQ < LogStash::Outputs::Base
     end
   end # def close
 
+  def receive(event)
+    @codec.encode(event)
+  end
+
   private
   def server?
     @mode == "server"
   end # def server?
 
-  public
-  def receive(event)
-    
-
-    @codec.encode(event)
-  end # def receive
-
   def publish(event, payload)
-    @logger.debug? && @logger.debug("0mq: sending", :event => payload)
     if @topology == "pubsub"
-      # TODO(sissel): Need to figure out how to fit this into the codecs system.
-      #@logger.debug("0mq output: setting topic to: #{event.sprintf(@topic)}")
-      #error_check(@zsocket.send_string(event.sprintf(@topic), ZMQ::SNDMORE),
-                  #"in topic send_string")
+      topic = event.sprintf(@topic)
+      error_check(@zsocket.send_string(topic, ZMQ::SNDMORE), "in topic send_string")
     end
+    @logger.debug? && @logger.debug("0mq: sending", :event => payload)
     error_check(@zsocket.send_string(payload), "in send_string")
   rescue => e
+    warn e.inspect
     @logger.warn("0mq output exception", :address => @address, :exception => e)
   end
+
+  def load_zmq
+    require "ffi-rzmq"
+    require "logstash/plugin_mixins/zeromq"
+    self.class.send(:include, LogStash::PluginMixins::ZeroMQ)
+  end
+
+  def connect
+    # Translate topology shorthand to socket types
+    case @topology
+    when "pair"
+      zmq_const = ZMQ::PAIR
+    when "pushpull"
+      zmq_const = ZMQ::PUSH
+    when "pubsub"
+      zmq_const = ZMQ::PUB
+    end # case socket_type
+
+    @zsocket = context.socket(zmq_const)
+
+    error_check(@zsocket.setsockopt(ZMQ::LINGER, 1),
+                "while setting ZMQ::LINGER == 1)")
+
+    if @sockopt
+      setopts(@zsocket, @sockopt)
+    end
+
+    @address.each do |addr|
+      setup(@zsocket, addr)
+    end
+  end
 end # class LogStash::Outputs::ZeroMQ

data/logstash-output-zeromq.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-zeromq'
-  s.version         = '2.0.4'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
-  s.summary         = "Write events to a 0MQ PUB socket."
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.summary         = "Write events to a 0MQ socket."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -21,9 +21,10 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
 
   s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency 'ffi-rzmq', '~> 2.0.4'
+  s.add_runtime_dependency 'logstash-mixin-zeromq', '>= 2.0.0'
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/integration/zeromq_spec.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/plugin"
+require "logstash/event"
+
+describe LogStash::Outputs::ZeroMQ, :integration => true do
+
+  describe "send events" do
+
+    let(:nevents)  { 10 }
+    let(:port)     { rand(1000)+1025 }
+
+    let(:conf) do
+      {  "address" => ["tcp://127.0.0.1:#{port}"],
+         "topology" => "pushpull" }
+    end
+
+    let(:events) do
+      output(conf, nevents) do
+        res = []
+        client = ZeroMQClient.new("127.0.0.1", port)
+        nevents.times do
+          res << client.recv
+        end
+        client.close
+        res
+      end
+    end
+
+    it "should receive the events" do
+      expect(events.count).to be(nevents)
+    end
+  end
+end

data/spec/outputs/zeromq_spec.rb

@@ -1 +1,34 @@
-require "logstash/devutils/rspec/spec_helper"
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/plugin"
+require "logstash/event"
+
+describe LogStash::Outputs::ZeroMQ, :zeromq => true do
+
+  context "when register and close" do
+
+    let(:plugin) { LogStash::Plugin.lookup("output", "zeromq").new({ "topology" => "pushpull" }) }
+
+    it "should register and close without errors" do
+      expect { plugin.register }.to_not raise_error
+      expect { plugin.do_close }.to_not raise_error
+    end
+
+  end
+
+  context "pubsub" do
+    let(:plugin) { LogStash::Plugin.lookup("output", "zeromq").new({"topology" => "pubsub", "topic" => "%{topic}"})}
+
+    before do
+      plugin.send(:load_zmq)
+    end
+
+    it "should use topic field as topic" do
+      mock_socket = instance_spy("ZMQ::Socket", :send_string => 0)
+      plugin.instance_variable_set(:@zsocket, mock_socket)
+      plugin.send(:publish, LogStash::Event.new("topic" => "test-topic", "message" => "text"), "payload")
+      expect(mock_socket).to have_received(:send_string).with("test-topic", ZMQ::SNDMORE).once.ordered
+      expect(mock_socket).to have_received(:send_string).with("payload").ordered
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/zeromq"
+require_relative "support/client"
+
+module ZeroMQHelpers
+
+  def setup_clients(number_of_clients, port)
+    number_of_clients.times.inject([]) do |clients|
+      clients << ZeroMQClient.new(localhost, port)
+    end
+  end
+
+  def output(config, size, &block)
+    plugin = LogStash::Plugin.lookup("output", "zeromq").new(config)
+    plugin.register
+    events = []
+    size.times do |n|
+      events << LogStash::Event.new({"message" => "data #{n}"})
+    end
+    pipeline_thread = Thread.new { plugin.multi_receive(events) }
+    sleep 0.3
+    result = block.call
+    plugin.close
+    pipeline_thread.join
+    result
+  end # def input
+
+end
+
+RSpec.configure do |config|
+  config.include ZeroMQHelpers
+  # config.filter_run_excluding({ :zeromq => true, :integration => true })
+  config.order = :random
+end

data/spec/support/client.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require 'ffi-rzmq'
+
+class ZeroMQClient
+
+  attr_reader :addr, :port, :context, :requester
+
+  def initialize(addr, port)
+    @addr   = addr
+    @port   = port
+    @context = ZMQ::Context.new(1)
+    @requester = context.socket(ZMQ::PULL)
+    @requester.bind("tcp://#{addr}:#{port}")
+  end
+
+  def recv
+    s = ''
+    @requester.recv_string s
+    s
+  end
+
+  def close
+    @requester.close
+    @context.terminate
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-zeromq
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +24,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0.beta2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0.beta2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -41,17 +61,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.4
-  name: ffi-rzmq
+        version: 2.0.0
+  name: logstash-mixin-zeromq
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.4
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -66,7 +86,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -79,9 +99,11 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/zeromq.rb
-- lib/logstash/util/zeromq.rb
 - logstash-output-zeromq.gemspec
+- spec/integration/zeromq_spec.rb
 - spec/outputs/zeromq_spec.rb
+- spec/spec_helper.rb
+- spec/support/client.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -107,6 +129,9 @@ rubyforge_project:
 rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
-summary: Write events to a 0MQ PUB socket.
+summary: Write events to a 0MQ socket.
 test_files:
+- spec/integration/zeromq_spec.rb
 - spec/outputs/zeromq_spec.rb
+- spec/spec_helper.rb
+- spec/support/client.rb

data/lib/logstash/util/zeromq.rb

@@ -1,47 +0,0 @@
-# encoding: utf-8
-require 'ffi-rzmq'
-require "logstash/namespace"
-
-module LogStash::Util::ZeroMQ
-  CONTEXT = ZMQ::Context.new
-  # LOGSTASH-400
-  # see https://github.com/chuckremes/ffi-rzmq/blob/master/lib/ffi-rzmq/socket.rb#L93-117
-  STRING_OPTS = %w{IDENTITY SUBSCRIBE UNSUBSCRIBE}
-
-  def context
-    CONTEXT
-  end
-
-  def setup(socket, address)
-    if server?
-      error_check(socket.bind(address), "binding to #{address}")
-    else
-      error_check(socket.connect(address), "connecting to #{address}")
-    end
-    @logger.info("0mq: #{server? ? 'connected' : 'bound'}", :address => address)
-  end
-
-  def error_check(rc, doing)
-    unless ZMQ::Util.resultcode_ok?(rc)
-      @logger.error("ZeroMQ error while #{doing}", { :error_code => rc })
-      raise "ZeroMQ Error while #{doing}"
-    end
-  end # def error_check
-
-  def setopts(socket, options)
-    options.each do |opt,value|
-      sockopt = opt.split('::')[1]
-      option = ZMQ.const_defined?(sockopt) ? ZMQ.const_get(sockopt) : ZMQ.const_missing(sockopt)
-      unless STRING_OPTS.include?(sockopt)
-        begin
-          Float(value)
-          value = value.to_i
-        rescue ArgumentError
-          raise "#{sockopt} requires a numeric value. #{value} is not numeric"
-        end
-      end # end unless
-      error_check(socket.setsockopt(option, value),
-              "while setting #{opt} == #{value}")
-    end # end each
-  end # end setopts
-end # module LogStash::Util::ZeroMQ