logstash-input-zeromq 2.0.4 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd37dfb55ab097ade7a77551197510e8a1155f9c
-  data.tar.gz: b53a58b848c7695f29b376222ff681eb43f13981
+  metadata.gz: 150429261e93e326c4af010caefb4ca3811d6624
+  data.tar.gz: 5af0979b436ffbfd933f38a2e78391355ed8f367
 SHA512:
-  metadata.gz: a054a6a1db8c599d2139d00701605ab91364cb552066a41f97f43a37656dd7a193b55bc56e8e265e56f3b55061f3d61a2ba9c38f885ef7797eaa08b02cbfd55f
-  data.tar.gz: ccf5d4c940a2220eb308600d711b77e8f65be0f2b4e85559564684db9afeea178ce65f9e6fa67647a0acc3f862fad1709767e0ea7081790c3f55f84b4e169715
+  metadata.gz: 6707e2ddb978dcca1deb114129f1970769a5fedb1a1532ac0dfb6b5c049a1a59afe5b5c28f3d0dd519efd85e200373863140ae8b7733bc36c852a5e68abda03b
+  data.tar.gz: c2b48f30b7652ed4e54fe529097d43dbcab0b64a26ae65abddfbefcf54a75a661709fd22899b4b5c6ca73d67b6685b9ad121d83069d6ff7f6d62ebd35afec763

data/CHANGELOG.md

@@ -1,9 +1,17 @@
+## 3.0.0
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+# 2.1.0
+  - Depend on logstash-mixin-zeromq, remove lib/logstash/util/zeromq
+  - Adapt test usage to be friendly with the way we run LS core default plugins test
+  - Fix plugin consumes 100% cpu
+  - Properly set topic field in pubsub topology
+  - Support multiple message frames
 # 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/CONTRIBUTORS

@@ -8,6 +8,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
+* Avishai Ish-Shalom (avishai-ish-shalom)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
-gemspec
+
+# Specify your gem's dependencies in logstash-mass_effect.gemspec
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-zeromq-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-zeromq-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-zeromq.svg)](https://travis-ci.org/logstash-plugins/logstash-input-zeromq)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/inputs/zeromq.rb

@@ -8,7 +8,7 @@ require "socket"
 # You need to have the 0mq 2.1.x library installed to be able to use
 # this input plugin.
 #
-# The default settings will create a subscriber binding to `tcp://127.0.0.1:2120` 
+# The default settings will create a subscriber binding to `tcp://127.0.0.1:2120`
 # waiting for connecting publishers.
 #
 class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
@@ -46,13 +46,19 @@ class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
   # You can specify multiple topics here
   config :topic, :validate => :array
 
+  # Event topic field
+  # This is used for the `pubsub` topology only
+  # When a message is received on a topic, the topic name on which
+  # the message was received will saved in this field.
+  config :topic_field, :validate => :string, :default => "topic"
+
   # mode
   # server mode binds/listens
   # client mode connects
   config :mode, :validate => ["server", "client"], :default => "server"
 
   # sender
-  # overrides the sender to 
+  # overrides the sender to
   # set the source of the event
   # default is `zmq+topology://type/`
   config :sender, :validate => :string
@@ -68,18 +74,30 @@ class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
   #  * `ZMQ::IDENTITY` - named queues
   #  * `ZMQ::SWAP_SIZE` - space for disk overflow
   #
-  # example: `sockopt => ["ZMQ::HWM", 50, "ZMQ::IDENTITY", "my_named_queue"]`
-  config :sockopt, :validate => :hash
+  # Example:
+  # [source,ruby]
+  #     sockopt => {
+  #        "ZMQ::HWM" => 50
+  #        "ZMQ::IDENTITY"  => "my_named_queue"
+  #     }
+  #
+  # defaults to: `sockopt => { "ZMQ::RCVTIMEO" => "1000" }`, which has the effect of "interrupting"
+  # the recv operation at least once every second to allow for properly shutdown handling.
+  config :sockopt, :validate => :hash, :default => { "ZMQ::RCVTIMEO" => "1000" }
 
   public
   def register
     require "ffi-rzmq"
-    require "logstash/util/zeromq"
-    self.class.send(:include, LogStash::Util::ZeroMQ)
+    require "logstash/plugin_mixins/zeromq"
+    self.class.send(:include, LogStash::PluginMixins::ZeroMQ)
+    @host = Socket.gethostname
+    init_socket
+  end # def register
 
+  def init_socket
     case @topology
     when "pair"
-      zmq_const = ZMQ::PAIR 
+      zmq_const = ZMQ::PAIR
     when "pushpull"
       zmq_const = ZMQ::PULL
     when "pubsub"
@@ -110,8 +128,7 @@ class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
         end
       end
     end
-
-  end # def register
+  end
 
   def close
     begin
@@ -127,36 +144,12 @@ class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
   end # def server?
 
   def run(output_queue)
-    host = Socket.gethostname
     begin
       while !stop?
-        # Here's the unified receiver
-        # Get the first part as the msg
-        m1 = ""
-        rc = @zsocket.recv_string(m1, ZMQ::DONTWAIT)
-        next if rc == -1 && ZMQ::Util.errno == ZMQ::EAGAIN
-        error_check(rc, "in recv_string")
-
-        @logger.debug("ZMQ receiving", :event => m1)
-        msg = m1
-        # If we have more parts, we'll eat the first as the topic
-        # and set the message to the second part
-        if @zsocket.more_parts?
-          @logger.debug("Multipart message detected. Setting @message to second part. First part was: #{m1}")
-          m2 = ''
-          rc2 = @zsocket.recv_string(m2)
-          error_check(rc2, "in recv_string")
-          @logger.debug("ZMQ receiving", :event => m2)
-          msg = m2
-        end
-        @codec.decode(msg) do |event|
-          event["host"] ||= host
-          decorate(event)
-          output_queue << event
-        end
+        handle_message(output_queue)
       end
     rescue => e
-      @logger.debug("ZMQ Error", :subscriber => @zsocket,
+      @logger.debug? && @logger.debug("ZMQ Error", :subscriber => @zsocket,
                     :exception => e)
       retry
     end # begin
@@ -167,4 +160,27 @@ class LogStash::Inputs::ZeroMQ < LogStash::Inputs::Base
     id = @address.first.clone
   end
 
+  def handle_message(output_queue)
+    # Here's the unified receiver
+    more = true
+    parts = []
+    rc = @zsocket.recv_strings(parts)
+    error_check(rc, "in recv_strings", true)
+    return unless ZMQ::Util.resultcode_ok?(rc)
+
+    if @topology == "pubsub" && parts.length > 1
+      # assume topic is a simple string
+      topic, *parts = parts
+    else
+      topic = nil
+    end
+    parts.each do |msg|
+      @codec.decode(msg) do |event|
+        event.set("host", event.get("host") || @host)
+        event.set(@topic_field, topic.force_encoding('UTF-8')) unless topic.nil?
+        decorate(event)
+        output_queue << event
+      end
+    end
+  end
 end # class LogStash::Inputs::ZeroMQ

data/logstash-input-zeromq.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-zeromq'
-  s.version         = '2.0.4'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "$summary"
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,9 +20,9 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-mixin-zeromq", ">= 2.0.1"
 
   s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency 'ffi-rzmq', '~> 2.0.4'
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/zeromq_spec.rb

@@ -2,6 +2,23 @@
 require_relative "../spec_helper"
 require "logstash/plugin"
 require "logstash/event"
+require "securerandom"
+
+def send_mock_messages(messages, &block)
+  socket = double("socket")
+  expect(socket).to receive(:recv_strings) do |arr|
+    messages.each do |msg|
+      msg.each do |frame|
+        arr << frame
+      end
+    end
+    0
+  end
+  plugin.instance_variable_set(:@zsocket, socket)
+  q = []
+  plugin.send(:handle_message, q)
+  q
+end
 
 describe LogStash::Inputs::ZeroMQ, :zeromq => true do
 
@@ -24,4 +41,39 @@ describe LogStash::Inputs::ZeroMQ, :zeromq => true do
     end
 
   end
+
+  context "pubsub" do
+    topic_field = SecureRandom.hex
+    let(:plugin) { LogStash::Plugin.lookup("input", "zeromq").new({"topology" => "pubsub", "topic_field" => topic_field}) }
+
+    before do
+      allow(plugin).to receive(:init_socket)
+      plugin.register
+    end
+
+    it "should set the topic field with multiple message frames" do
+      events = send_mock_messages([["topic", '{"message": "message"}', '{"message": "message2"}']])
+      expect(events.first.get(topic_field)).to eq("topic")
+      expect(events.first.get("message")).to eq("message")
+      expect(events[1].get("message")).to eq("message2")
+      expect(events[1].get(topic_field)).to eq("topic")
+      expect(events.length).to eq(2)
+    end
+  end
+
+  context "pushpull" do
+    let(:plugin) { LogStash::Plugin.lookup("input", "zeromq").new({ "topology" => "pushpull" }) }
+
+    before do
+      allow(plugin).to receive(:init_socket)
+      plugin.register
+    end
+
+    it "should receive multiple frames" do
+      events = send_mock_messages([['{"message": "message"}', '{"message": "message2"}']])
+      expect(events.first.get("message")).to eq("message")
+      expect(events[1].get("message")).to eq("message2")
+      expect(events.length).to eq(2)
+    end
+  end
 end

data/spec/integration/zeromq_spec.rb

@@ -2,9 +2,12 @@
 require_relative "../spec_helper"
 require "logstash/plugin"
 require "logstash/event"
+require "json"
 
 describe LogStash::Inputs::ZeroMQ, :integration => true do
 
+  let(:helpers) { ZeroMQHelpers.new }
+
   describe "receive events" do
 
     let(:nevents)  { 10 }
@@ -16,10 +19,11 @@ describe LogStash::Inputs::ZeroMQ, :integration => true do
     end
 
     let(:events) do
-      input(conf, nevents) do
+      helpers.input(conf, nevents) do
         client = ZeroMQClient.new("127.0.0.1", port)
         nevents.times do |value|
-          client.send("data #{value}")
+          client.send("TOPIC", ZMQ::SNDMORE)
+          client.send({"message" => "data #{value}"}.to_json)
         end
         client.close
       end
@@ -27,6 +31,7 @@ describe LogStash::Inputs::ZeroMQ, :integration => true do
 
     it "should receive the events" do
       expect(events.count).to be(nevents)
+      expect(events.map(&:to_hash)).to all(include("topic" => "TOPIC"))
     end
   end
 end

data/spec/spec_helper.rb

@@ -3,13 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/zeromq"
 require_relative "support/client"
 
-module ZeroMQHelpers
-
-  def setup_clients(number_of_clients, port)
-    number_of_clients.times.inject([]) do |clients|
-      clients << ZeroMQClient.new(localhost, port)
-    end
-  end
+class ZeroMQHelpers
 
   def input(config, size, &block)
     plugin = LogStash::Plugin.lookup("input", "zeromq").new(config)
@@ -31,7 +25,6 @@ module ZeroMQHelpers
 end
 
 RSpec.configure do |config|
-  config.include ZeroMQHelpers
-  config.filter_run_excluding({ :zeromq => true, :integration => true })
+  # config.filter_run_excluding({ :zeromq => true, :integration => true })
   config.order = :random
 end

data/spec/support/client.rb

@@ -13,8 +13,8 @@ class ZeroMQClient
     @requester.connect("tcp://#{addr}:#{port}")
   end
 
-  def send(data)
-    @requester.send_string(data)
+  def send(data, flags=0)
+    @requester.send_string(data, flags)
   end
 
   def close

metadata

@@ -1,72 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-zeromq
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-core-plugin-api
-  prerelease: false
+        version: '2.0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-zeromq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-codec-json
-  prerelease: false
+        version: 2.0.1
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.4
-  name: ffi-rzmq
-  prerelease: false
+        version: '0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.4
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -79,7 +81,6 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/inputs/zeromq.rb
-- lib/logstash/util/zeromq.rb
 - logstash-input-zeromq.gemspec
 - spec/inputs/zeromq_spec.rb
 - spec/integration/zeromq_spec.rb
@@ -91,7 +92,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -106,9 +107,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
 summary: "$summary"
 test_files:

data/lib/logstash/util/zeromq.rb

@@ -1,46 +0,0 @@
-# encoding: utf-8
-require 'ffi-rzmq'
-require "logstash/namespace"
-
-module LogStash::Util::ZeroMQ
-  # LOGSTASH-400
-  # see https://github.com/chuckremes/ffi-rzmq/blob/master/lib/ffi-rzmq/socket.rb#L93-117
-  STRING_OPTS = %w{IDENTITY SUBSCRIBE UNSUBSCRIBE}
-
-  def context
-    @context ||= ZMQ::Context.new
-  end
-
-  def setup(socket, address)
-    if server?
-      error_check(socket.bind(address), "binding to #{address}")
-    else
-      error_check(socket.connect(address), "connecting to #{address}")
-    end
-    @logger.info("0mq: #{server? ? 'connected' : 'bound'}", :address => address)
-  end
-
-  def error_check(rc, doing)
-    unless ZMQ::Util.resultcode_ok?(rc)
-      @logger.error("ZeroMQ error while #{doing}", { :error_code => rc })
-      raise "ZeroMQ Error while #{doing}"
-    end
-  end # def error_check
-
-  def setopts(socket, options)
-    options.each do |opt,value|
-      sockopt = opt.split('::')[1]
-      option = ZMQ.const_defined?(sockopt) ? ZMQ.const_get(sockopt) : ZMQ.const_missing(sockopt)
-      unless STRING_OPTS.include?(sockopt)
-        begin
-          Float(value)
-          value = value.to_i
-        rescue ArgumentError
-          raise "#{sockopt} requires a numeric value. #{value} is not numeric"
-        end
-      end # end unless
-      error_check(socket.setsockopt(option, value),
-              "while setting #{opt} == #{value}")
-    end # end each
-  end # end setopts
-end # module LogStash::Util::ZeroMQ