logstash-input-snmp 1.2.6 → 1.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 149791948ca146c248ba111642501748a2c0eacedc6b5774e4c8dd02ce9fb746
-  data.tar.gz: e5ac1d8d8d52481ea6b57a37ec09428a36b88c4c374c379aad3d5950c326e24a
+  metadata.gz: 4bcb5f73151249317b6f7b36c028a3f863e7d4a88ba6da977fbc88d255355364
+  data.tar.gz: 85ef765af5f93aefb92a8f40d9738101ada3b5173bb9b4b8de3d4ec21a406578
 SHA512:
-  metadata.gz: cfa32f9ecea52e10fcf38ece4ac80f2037da7a3dfed55560decba78eb80bdfc559a22952d8d85b83e44fbedcbe667b5f01d21d037406eea55a09fa5a684d4cee
-  data.tar.gz: ec8eca95c464381397b581260803d238d1fb1bf57bd7ae8add5d2718aa438509f0be70473592ba07bb12d19abe5a8aaff49cb5208b010aff283e92782a91c4a4
+  metadata.gz: 423f10e23a58b04a72c47d70beb06053dedc8d07e1ec1e36c835ca783c6791bb6ccd12d701d3e50bd4ebe173185a4240ae21a93c36d7ecdf38bf4e2ae6d58f25
+  data.tar.gz: f452759e6c453d088b002721ae1520b1c22291e16ff609d8f308e57c3e9aa9dc29b008c809a9a6a76e76b00e68c0399fc96b63f23dc066aa403a682f19712f2e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.2.8
+  - Fixed interval handling to only sleep off the _remainder_ of the interval (if any), and to log a helpful warning when crawling the hosts takes longer than the configured interval [#61](https://github.com/logstash-plugins/logstash-input-snmp/issues/61)
+
+## 1.2.7
+  - Added integration tests to ensure SNMP server and IPv6 connections [#87](https://github.com/logstash-plugins/logstash-input-snmp/pull/87)
+
 ## 1.2.6
   - Docs: example on setting IPv6 hosts [#89](https://github.com/logstash-plugins/logstash-input-snmp/pull/89)
 

data/docs/index.asciidoc

@@ -163,6 +163,7 @@ input {
 ===== `interval` 
 
 The `interval` option specifies the polling interval in seconds.
+If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs.
 
 * Value type is <<number,number>>
 * Default value is `30`

data/lib/logstash/inputs/snmp.rb

@@ -161,9 +161,9 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
   end
 
   def run(queue)
-    # for now a naive single threaded poller which sleeps for the given interval between
+    # for now a naive single threaded poller which sleeps off the remaining interval between
     # each run. each run polls all the defined hosts for the get and walk options.
-    while !stop?
+    stoppable_interval_runner.every(@interval, "polling hosts") do
       @client_definitions.each do |definition|
         result = {}
         if !definition[:get].empty?
@@ -207,11 +207,13 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
           queue << event
         end
       end
-
-      Stud.stoppable_sleep(@interval) { stop? }
     end
   end
 
+  def stoppable_interval_runner
+    StoppableIntervalRunner.new(self)
+  end
+
   def close
     @client_definitions.each do |definition|
       begin
@@ -222,6 +224,9 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
     end
   end
 
+  def stop
+  end
+
   private
 
   OID_REGEX = /^\.?([0-9\.]+)$/
@@ -295,4 +300,45 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
   def validate_strip!
     raise(LogStash::ConfigurationError, "you can not specify both oid_root_skip and oid_path_length") if @oid_root_skip > 0 and @oid_path_length > 0
   end
+
+  ##
+  # The StoppableIntervalRunner is capable of running a block of code at a
+  # repeating interval, while respecting the stop condition of the plugin.
+  class StoppableIntervalRunner
+    ##
+    # @param plugin [#logger,#stop?]
+    def initialize(plugin)
+      @plugin = plugin
+    end
+
+    ##
+    # Runs the provided block repeatedly using the provided interval.
+    # After executing the block, the remainder of the interval if any is slept off
+    # using an interruptible sleep.
+    # If no time remains, a warning is emitted to the logs.
+    #
+    # @param interval_seconds [Integer,Float]
+    # @param desc [String] (default: "operation"): a description to use when logging
+    # @yield
+    def every(interval_seconds, desc="operation", &block)
+      until @plugin.stop?
+        start_time = Time.now
+
+        yield
+
+        duration_seconds = Time.now - start_time
+        if duration_seconds >= interval_seconds
+          @plugin.logger.warn("#{desc} took longer than the configured interval", :interval_seconds => interval_seconds, :duration_seconds => duration_seconds.round(3))
+        else
+          remaining_interval = interval_seconds - duration_seconds
+          sleep(remaining_interval)
+        end
+      end
+    end
+
+    # @api private
+    def sleep(duration)
+      Stud.stoppable_sleep(duration) { @plugin.stop? }
+    end
+  end
 end

data/logstash-input-snmp.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-snmp'
-  s.version       = '1.2.6'
+  s.version       = '1.2.8'
   s.licenses      = ['Apache-2.0']
   s.summary       = "SNMP input plugin"
   s.description   = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/integration/it_spec.rb

@@ -0,0 +1,203 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/snmp"
+
+describe LogStash::Inputs::Snmp do
+  let(:config) { {"get" => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"]} }
+  let(:plugin) { LogStash::Inputs::Snmp.new(config)}
+
+  shared_examples "snmp plugin return single event" do
+    it "should have OID value" do
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+      event = queue.pop
+
+      expect(event).to be_a(LogStash::Event)
+      expect(event.get("iso.org.dod.internet.mgmt.mib-2.system.sysUpTime.sysUpTimeInstance")).to be_a Integer
+      expect(event.get("iso.org.dod.internet.mgmt.mib-2.system.sysName.0")).to be_a String
+      expect(event.get("iso.org.dod.internet.mgmt.mib-2.system.sysDescr.0")).to be_a String
+    end
+  end
+
+  shared_examples "snmp plugin return one udp event and one tcp event" do |config|
+    it "should have one udp from snmp1 and one tcp from snmp2" do
+      events = input(config) { |_, queue| 2.times.collect { queue.pop } }
+      udp = 0; tcp = 0
+      events.each { |event|
+        if event.get("[@metadata][host_protocol]") == "udp"
+          udp += 1
+          expect(event.get("[@metadata][host_protocol]")).to eq("udp")
+          expect(event.get("[@metadata][host_address]")).to eq("snmp1")
+          expect(event.get("[@metadata][host_port]")).to eq("161")
+        else
+          tcp += 1
+          expect(event.get("[@metadata][host_protocol]")).to eq("tcp")
+          expect(event.get("[@metadata][host_address]")).to eq("snmp2")
+          expect(event.get("[@metadata][host_port]")).to eq("162")
+        end
+      }
+      expect(udp).to eq(1)
+      expect(tcp).to eq(1)
+    end
+  end
+
+  describe "against single snmp server with snmpv2 and udp", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}]})}
+    it_behaves_like "snmp plugin return single event"
+  end
+
+  describe "against single server with snmpv3 and tcp", :integration => true do
+    let(:config) { super().merge({
+       "hosts" => [{"host" => "tcp:snmp1/161", "version" => "3"}],
+       "security_name" => "user_1",
+       "auth_protocol" => "sha",
+       "auth_pass" => "STrP@SSPhr@sE",
+       "priv_protocol" => "aes",
+       "priv_pass" => "STr0ngP@SSWRD161",
+       "security_level" => "authPriv"
+                               })}
+
+    it_behaves_like "snmp plugin return single event"
+  end
+
+  describe "invalid user against snmpv3 server", :integration => true do
+    let(:config) { super().merge({
+                                   "hosts" => [{"host" => "tcp:snmp1/161", "version" => "3"}],
+                                   "security_name" => "user_2",
+                                   "auth_protocol" => "sha",
+                                   "auth_pass" => "STrP@SSPhr@sE",
+                                   "priv_protocol" => "aes",
+                                   "priv_pass" => "STr0ngP@SSWRD161",
+                                   "security_level" => "authPriv"
+                               })}
+
+    it "should have error log" do
+      expect(plugin.logger).to receive(:error).once
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+    end
+  end
+
+  describe "single input plugin on single server with snmpv2 and mix of udp and tcp", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}, {"host" => "tcp:snmp1/161", "community" => "public"}]})}
+    it "should return two events " do
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+
+      host_cnt_snmp1 = queue.select {|event| event.get("host") == "snmp1"}.size
+      expect(queue.size).to eq(2)
+      expect(host_cnt_snmp1).to eq(2)
+    end
+  end
+
+  describe "single input plugin on multiple udp hosts", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}, {"host" => "udp:snmp2/162", "community" => "public"}]})}
+    it "should return two events, one per host" do
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+
+      hosts = queue.map { |event| event.get("host") }.sort
+      expect(queue.size).to eq(2)
+      expect(hosts).to eq(["snmp1", "snmp2"])
+    end
+  end
+
+  describe "multiple pipelines and mix of udp tcp hosts", :integration => true do
+    let(:config) { {"get" => ["1.3.6.1.2.1.1.1.0"], "hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}]} }
+    let(:config2) { {"get" => ["1.3.6.1.2.1.1.1.0"], "hosts" => [{"host" => "tcp:snmp2/162", "community" => "public"}]} }
+    let(:plugin) { LogStash::Inputs::Snmp.new(config)}
+    let(:plugin2) { LogStash::Inputs::Snmp.new(config2)}
+
+    it "should return two events, one per host" do
+      plugin.register
+      plugin2.register
+      queue = []
+      queue2 = []
+      t = Thread.new {
+        stop_plugin_after_seconds(plugin)
+        plugin.run(queue)
+      }
+      t2 = Thread.new {
+        stop_plugin_after_seconds(plugin2)
+        plugin2.run(queue2)
+      }
+      t.join(2100)
+      t2.join(2100)
+      plugin.close
+      plugin2.close
+
+      hosts = [queue.pop, queue2.pop].map { |event| event.get("host") }.sort
+      expect(hosts).to eq(["snmp1", "snmp2"])
+    end
+  end
+
+  describe "multiple plugin inputs and mix of udp tcp hosts", :integration => true do
+    config = <<-CONFIG
+        input {
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "udp:snmp1/161" community => "public"}]
+          }
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "tcp:snmp2/162" community => "public"}]
+          }
+        }
+    CONFIG
+
+    it_behaves_like "snmp plugin return one udp event and one tcp event", config
+  end
+
+  describe "two plugins on different hosts with snmpv3 with same security name with different credentials and mix of udp and tcp", :integration => true do
+    config = <<-CONFIG
+        input {
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "udp:snmp1/161" version => "3"}]
+            security_name => "user_1"
+            auth_protocol => "sha"
+            auth_pass => "STrP@SSPhr@sE"
+            priv_protocol => "aes"
+            priv_pass => "STr0ngP@SSWRD161"
+            security_level => "authPriv"
+          }
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "tcp:snmp2/162" version => "3"}]
+            security_name => "user_1"
+            auth_protocol => "sha"
+            auth_pass => "STrP@SSPhr@sE"
+            priv_protocol => "aes"
+            priv_pass => "STr0ngP@SSWRD162"
+            security_level => "authPriv"
+          }
+        }
+    CONFIG
+
+    it_behaves_like "snmp plugin return one udp event and one tcp event", config
+  end
+
+  describe "single host with tcp over ipv6", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "tcp:[2001:3984:3989::161]/161"}]})}
+    it_behaves_like "snmp plugin return single event"
+  end
+
+  def stop_plugin_after_seconds(plugin)
+      Thread.new{
+        sleep(2)
+        plugin.do_stop
+      }
+  end
+
+end

data/spec/inputs/snmp_spec.rb

@@ -171,5 +171,95 @@ describe LogStash::Inputs::Snmp do
       expect(event.get("host")).to eq("udp:127.0.0.1/161,public")
     end
   end
+
+  context "StoppableIntervalRunner" do
+    let(:stop_holder) { Struct.new(:value).new(false) }
+
+    before(:each) do
+      allow(plugin).to receive(:stop?) { stop_holder.value }
+    end
+
+    let(:plugin) do
+      double("Plugin").tap do |dbl|
+        allow(dbl).to receive(:logger).and_return(double("Logger").as_null_object)
+        allow(dbl).to receive(:stop?) { stop_holder.value }
+      end
+    end
+
+    subject(:interval_runner) { LogStash::Inputs::Snmp::StoppableIntervalRunner.new(plugin) }
+
+    context "#every" do
+      context "when the plugin is stopped" do
+        let(:interval_seconds) { 2 }
+        it 'does not yield the block' do
+          stop_holder.value = true
+          expect { |yielder| interval_runner.every(interval_seconds, &yielder) }.to_not yield_control
+        end
+      end
+
+      context "when the yield takes shorter than the interval" do
+        let(:duration_seconds) { 1 }
+        let(:interval_seconds) { 2 }
+
+        it 'sleeps off the remainder' do
+          allow(interval_runner).to receive(:sleep).and_call_original
+
+          interval_runner.every(interval_seconds) do
+            Kernel::sleep(duration_seconds) # non-stoppable
+            stop_holder.value = true # prevent re-runs
+          end
+
+          expect(interval_runner).to have_received(:sleep).with(a_value_within(0.1).of(1))
+        end
+      end
+
+      context "when the yield takes longer than the interval" do
+        let(:duration_seconds) { 2 }
+        let(:interval_seconds) { 1 }
+
+        it 'logs a warning to the plugin' do
+          allow(interval_runner).to receive(:sleep).and_call_original
+
+          interval_runner.every(interval_seconds) do
+            Kernel::sleep(duration_seconds) # non-stoppable
+            stop_holder.value = true # prevent re-runs
+          end
+
+          expect(interval_runner).to_not have_received(:sleep)
+
+          expect(plugin.logger).to have_received(:warn).with(a_string_including("took longer"), a_hash_including(:interval_seconds => interval_seconds, :duration_seconds => a_value_within(0.1).of(duration_seconds)))
+        end
+      end
+
+      it 'runs regularly until the plugin is stopped' do
+        timestamps = []
+
+        thread = Thread.new do
+          interval_runner.every(1) do
+            timestamps << Time.now
+            Kernel::sleep(Random::rand(0.8))
+          end
+        end
+
+        Kernel::sleep(5)
+        expect(thread).to be_alive
+
+        stop_holder.value = true
+        Kernel::sleep(1)
+
+        aggregate_failures do
+          expect(thread).to_not be_alive
+          expect(timestamps.count).to be_within(1).of(5)
+
+          timestamps.each_cons(2) do |previous, current|
+            # ensure each start time is very close to 1s after the previous.
+            expect(current - previous).to be_within(0.05).of(1)
+          end
+
+          thread.kill if thread.alive?
+        end
+      end
+    end
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-snmp
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.2.8
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-21 00:00:00.000000000 Z
+date: 2021-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -414,9 +414,12 @@ files:
 - logstash-input-snmp.gemspec
 - spec/fixtures/RFC1213-MIB.dic
 - spec/fixtures/collision.dic
+- spec/inputs/integration/it_spec.rb
 - spec/inputs/snmp/base_client_spec.rb
+- spec/inputs/snmp/interval_runner_spec.rb
 - spec/inputs/snmp/mib_spec.rb
 - spec/inputs/snmp_spec.rb
+- vendor/jar-dependencies/org/snmp4j/snmp4j/2.5.11/snmp4j-2.5.11.jar
 - vendor/jar-dependencies/org/snmp4j/snmp4j/2.8.4/snmp4j-2.8.4.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -440,14 +443,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: SNMP input plugin
 test_files:
 - spec/fixtures/RFC1213-MIB.dic
 - spec/fixtures/collision.dic
+- spec/inputs/integration/it_spec.rb
 - spec/inputs/snmp/base_client_spec.rb
+- spec/inputs/snmp/interval_runner_spec.rb
 - spec/inputs/snmp/mib_spec.rb
 - spec/inputs/snmp_spec.rb