logstash-input-snmp 1.2.6 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 149791948ca146c248ba111642501748a2c0eacedc6b5774e4c8dd02ce9fb746
-  data.tar.gz: e5ac1d8d8d52481ea6b57a37ec09428a36b88c4c374c379aad3d5950c326e24a
+  metadata.gz: 13fdcfbd3e486a466a7fa32e1de3e6bbdccd30779f5eb4fce9ef8645355c8322
+  data.tar.gz: b8e80f54fbe399f262e48aff1fd614d578b16d493cded3e15446593ebffd1f93
 SHA512:
-  metadata.gz: cfa32f9ecea52e10fcf38ece4ac80f2037da7a3dfed55560decba78eb80bdfc559a22952d8d85b83e44fbedcbe667b5f01d21d037406eea55a09fa5a684d4cee
-  data.tar.gz: ec8eca95c464381397b581260803d238d1fb1bf57bd7ae8add5d2718aa438509f0be70473592ba07bb12d19abe5a8aaff49cb5208b010aff283e92782a91c4a4
+  metadata.gz: fc44139cfc59066ffa89fc916903e4507972b9650cc056ee0bb897a8028dcf93abf30c07627926ce717a0da275d084ebf45b7e631355d04752ef55e868eef3f7
+  data.tar.gz: 1ca064497bd573c6d3de4c00bf0d744fb14339926d61203d6a7e74fcbb6887c64f784e32ecb579db11fc2ff849c329b7479df92a44b24803d653b316ab41cbd6

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 1.3.1
+  -  Refactor: handle no response(s) wout error logging [#105](https://github.com/logstash-plugins/logstash-input-snmp/pull/105)
+
+## 1.3.0
+  - Feat: ECS compliance + optional target [#99](https://github.com/logstash-plugins/logstash-input-snmp/pull/99)
+  - Internal: update to Gradle 7 [#102](https://github.com/logstash-plugins/logstash-input-snmp/pull/102)
+
+## 1.2.8
+  - Fixed interval handling to only sleep off the _remainder_ of the interval (if any), and to log a helpful warning when crawling the hosts takes longer than the configured interval [#100](https://github.com/logstash-plugins/logstash-input-snmp/pull/100). Fixes [#61](https://github.com/logstash-plugins/logstash-input-snmp/issues/61).
+
+## 1.2.7
+  - Added integration tests to ensure SNMP server and IPv6 connections [#90](https://github.com/logstash-plugins/logstash-input-snmp/issues/90). Fixes[#87](https://github.com/logstash-plugins/logstash-input-snmp/issues/87).
+
 ## 1.2.6
   - Docs: example on setting IPv6 hosts [#89](https://github.com/logstash-plugins/logstash-input-snmp/pull/89)
 

data/CONTRIBUTORS

@@ -4,6 +4,7 @@ reports, or in general have helped logstash along its way.
 Contributors:
 * Colin Surprenant - colin.surprenant@gmail.com
 * Dan Major - axrayn@gmail.com
+* Patrick Prugger - pprugger@gmx.at
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/docs/index.asciidoc

@@ -26,6 +26,22 @@ to gather information related to the current state of the devices operation.
 
 The SNMP input plugin supports SNMP v1, v2c, and v3 over UDP and TCP transport protocols.
 
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+Because SNMP data has specific field names based on OIDs, we recommend setting a <<plugins-{type}s-{plugin}-target>>.
+Metadata fields follow a specific naming convention when <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled.
+
+[cols="<l,<l,e,<e"]
+|=======================================================================
+|ECS disabled |ECS v1, v8 |Description
+|[@metadata][host_protocol]    |[@metadata][input][snmp][host][protocol]    |The protocol used to retrieve data e.g. "udp"
+|[@metadata][host_address]     |[@metadata][input][snmp][host][address]     |The host IP e.g. "192.168.1.1"
+|[@metadata][host_port]        |[@metadata][input][snmp][host][port]        |The host's port e.g. "161"
+|[@metadata][host_community]   |[@metadata][input][snmp][host][community]   |The configured community e.g. "public"
+|[host]                        |[host][ip]                                  |Same as `[@metadata][host_address]`, host's IP address
+|=======================================================================
+
 [id="plugins-{type}s-{plugin}-import-mibs"]
 ==== Importing MIBs 
 
@@ -55,6 +71,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-get>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
@@ -63,6 +80,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-oid_path_length>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-walk>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-tables>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
 
 ==== SNMPv3 Authentication Options
@@ -86,14 +104,14 @@ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options suppo
 input plugins.
 
 [id="plugins-{type}s-{plugin}-get"]
-===== `get` 
+===== `get`
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting
 
 Use the `get` option to query for scalar values for the given OID(s).
 One or more OID(s) are specified as an array of strings of OID(s).
 
-* Value type is <<array,array>>
-* There is no default value for this setting
-
 Example
 [source,ruby]
 -----
@@ -108,14 +126,14 @@ input {
 [id="plugins-{type}s-{plugin}-hosts"]
 ===== `hosts`
 
+  * Value type is <<array,array>>
+  * There is no default value for this setting
+
 The `hosts` option specifies the list of hosts to query the configured `get` and `walk` options.
 
 Each host definition is a hash and must define the `host` key and value.
 `host` must use the format `{tcp|udp}:{ip address}/{port}`, for example `host => "udp:127.0.0.1/161"`
 
-* Value type is <<array,array>>
-* There is no default value for this setting
-
 Each host definition can optionally include the following keys and values:
 
 * `community` the community string, default is `public`.
@@ -160,56 +178,58 @@ input {
 -----
 
 [id="plugins-{type}s-{plugin}-interval"]
-===== `interval` 
+===== `interval`
 
-The `interval` option specifies the polling interval in seconds.
+  * Value type is <<number,number>>
+  * Default value is `30`
 
-* Value type is <<number,number>>
-* Default value is `30`
+The `interval` option specifies the polling interval in seconds.
+If polling all configured hosts takes longer than this interval, a warning will be emitted to the logs.
 
 [id="plugins-{type}s-{plugin}-mib_paths"]
-===== `mib_paths` 
+===== `mib_paths`
 
-The `mib_paths` option specifies the location of one or more imported MIB files. The value can be either a dir path containing
-the imported MIB `.dic` files or a file path to a single MIB `.dic` file.
+  * Value type is <<path,path>>
+  * There is no default value for this setting
 
-* Value type is <<path,path>>
-* There is no default value for this setting
+The `mib_paths` option specifies the location of one or more imported MIB files.
+The value can be either a dir path containing the imported MIB `.dic` files or a
+file path to a single MIB `.dic` file.
 
 This plugin includes the IETF MIBs.
 If you require other MIBs, you need to import them. See <<plugins-{type}s-{plugin}-import-mibs>>.
 
 [id="plugins-{type}s-{plugin}-oid_root_skip"]
-===== `oid_root_skip` 
+===== `oid_root_skip`
+
+  * Value type is <<number,number>>
+  * Default value is `0`
 
 The `oid_root_skip` option specifies the number of OID root digits to ignore in the event field name.
 For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the first 5 digits could be ignored by setting `oid_root_skip => 5`
 which would result in a field name "1.1.1.0". Similarly when a MIB is used an OID such
 "1.3.6.1.2.mib-2.system.sysDescr.0" would become "mib-2.system.sysDescr.0"
 
-* Value type is <<number,number>>
-* Default value is `0`
-
 [id="plugins-{type}s-{plugin}-oid_path_length"]
 ===== `oid_path_length`
 
+  * Value type is <<number,number>>
+  * Default value is `0`
+
 The `oid_path_length` option specifies the number of OID root digits to retain in the event field name.
 For example, in a numeric OID like "1.3.6.1.2.1.1.1.0" the last 2 digits could be retained by setting `oid_path_length => 2`
 which would result in a field name "1.0". Similarly when a MIB is used an OID such
 "1.3.6.1.2.mib-2.system.sysDescr.0" would become "sysDescr.0"
 
-* Value type is <<number,number>>
-* Default value is `0`
-
 [id="plugins-{type}s-{plugin}-walk"]
 ===== `walk`
 
+  * Value type is <<array,array>>
+  * There is no default value for this setting
+
 Use the `walk` option to retrieve the subtree of information for the given OID(s).
 One or more OID(s) are specified as an array of strings of OID(s).
 
-* Value type is <<array,array>>
-* There is no default value for this setting
-
 Queries the subtree of information starting at the given OID(s).
 
 Example
@@ -225,14 +245,14 @@ Example
 [id="plugins-{type}s-{plugin}-tables"]
 ===== `tables`
 
+  * Value type is <<array,array>>
+  * There is no default value for this setting
+  * Results are returned under a field using the table name
+
 The `tables` option is used to query for tabular values for the given column OID(s).
 
 Each table definition is a hash and must define the name key and value and the columns to return.
 
-* Value type is <<array,array>>
-* There is no default value for this setting
-* Results are returned under a field using the table name
-
 *Specifying a single table*
 
 [source,ruby]
@@ -266,10 +286,10 @@ These options are required only if you are using SNMPv3.
 [id="plugins-{type}s-{plugin}-auth_pass"]
 ===== `auth_pass`
 
-The `auth_pass` option specifies the SNMPv3 authentication passphrase or password
+  * Value type is <<password,password>>
+  * There is no default value for this setting
 
-* Value type is <<password,password>>
-* There is no default value for this setting
+The `auth_pass` option specifies the SNMPv3 authentication passphrase or password.
 
 [id="plugins-{type}s-{plugin}-auth_protocol"]
 ===== `auth_protocol`
@@ -279,38 +299,66 @@ The `auth_protocol` option specifies the SNMPv3 authentication protocol or type
 * Value can be any of: `md5`, `sha`, `sha2`, `hmac128sha224`, `hmac192sha256`, `hmac256sha384`, `hmac384sha512`
 * There is no default value for this setting
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event)
+    ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `host` field)
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+
 [id="plugins-{type}s-{plugin}-priv_pass"]
 ===== `priv_pass`
 
-The `priv_pass` option specifies the SNMPv3 encryption password
+  * Value type is <<password,password>>
+  * There is no default value for this setting
 
-* Value type is <<password,password>>
-* There is no default value for this setting
+The `priv_pass` option specifies the SNMPv3 encryption password.
 
 [id="plugins-{type}s-{plugin}-priv_protocol"]
 ===== `priv_protocol`
 
-The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol.
+  * Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256`
+  * Note that `aes` and `aes128` are equivalent
+  * There is no default value for this setting
 
-* Value can be any of: `des`, `3des`, `aes`, `aes128`, `aes192`, `aes256`
-* Note that `aes` and `aes128` are equivalent
-* There is no default value for this setting
+The `priv_protocol` option specifies the SNMPv3 privacy/encryption protocol.
 
 [id="plugins-{type}s-{plugin}-security_name"]
 ===== `security_name`
 
-The `security_name` option specifies the SNMPv3 security name or user name
+  * Value type is <<string,string>>
+  * There is no default value for this setting
 
-* Value type is <<string,string>>
-* There is no default value for this setting
+The `security_name` option specifies the SNMPv3 security name or user name.
 
 [id="plugins-{type}s-{plugin}-security_level"]
 ===== `security_level`
 
-The `security_level` option specifies the SNMPv3 security level between Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy
+  * Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv`
+  * There is no default value for this setting
 
-* Value can be any of: `noAuthNoPriv`, `authNoPriv`, `authPriv`
-* There is no default value for this setting
+The `security_level` option specifies the SNMPv3 security level between
+Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy. 
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting
+
+The name of the field under which SNMP payloads are assigned.
+If not specified data will be stored in the root of the event.
+
+Setting a target is recommended when <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled.
+
+[id="plugins-{type}s-{plugin}-examples"]
+==== Configuration examples
 
 *Specifying SNMPv3 settings*
 
@@ -331,8 +379,6 @@ input {
 
 -----
 
-==== More configuration examples
-
 *Using both `get` and `walk` in the same poll cycle for each host(s)*
 
 [source,ruby]

data/lib/logstash/inputs/snmp/base_client.rb

@@ -52,13 +52,13 @@ module LogStash
     end
 
     def walk(oid, strip_root = 0, path_length = 0)
-      result = {}
-
       pdufactory = get_pdu_factory
       treeUtils = TreeUtils.new(@snmp, pdufactory)
       events = treeUtils.getSubtree(@target, OID.new(oid))
       return nil if events.nil? || events.size == 0
 
+      result = {}
+
       events.each do |event|
         next if event.nil?
 

data/lib/logstash/inputs/snmp.rb

@@ -7,11 +7,24 @@ require_relative "snmp/client"
 require_relative "snmp/clientv3"
 require_relative "snmp/mib"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+
 # Generate a repeating message.
 #
 # This plugin is intented only as an example.
 
 class LogStash::Inputs::Snmp < LogStash::Inputs::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "snmp"
 
   # List of OIDs for which we want to retrieve the scalar value
@@ -67,9 +80,6 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
   # The default, `30`, means poll each host every 30 seconds.
   config :interval, :validate => :number, :default => 30
 
-  # Add the default "host" field to the event.
-  config :add_field, :validate => :hash, :default => { "host" => "%{[@metadata][host_address]}" }
-
   # SNMPv3 Credentials
   #
   # A single user can be configured and will be used for all defined SNMPv3 hosts.
@@ -94,9 +104,29 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
   # The SNMPv3 security level can be Authentication, No Privacy; Authentication, Privacy; or no Authentication, no Privacy
   config :security_level, :validate => ["noAuthNoPriv", "authNoPriv", "authPriv"]
 
+  # Defines a target field for placing fields.
+  # If this setting is omitted, data gets stored at the root (top level) of the event.
+  # The target is only relevant while decoding data into a new event.
+  config :target, :validate => :field_reference
+
   BASE_MIB_PATH = ::File.join(__FILE__, "..", "..", "..", "mibs")
   PROVIDED_MIB_PATHS = [::File.join(BASE_MIB_PATH, "logstash"), ::File.join(BASE_MIB_PATH, "ietf")].map { |path| ::File.expand_path(path) }
 
+  def initialize(params={})
+    super(params)
+
+    @host_protocol_field = ecs_select[disabled: '[@metadata][host_protocol]', v1: '[@metadata][input][snmp][host][protocol]']
+    @host_address_field = ecs_select[disabled: '[@metadata][host_address]', v1: '[@metadata][input][snmp][host][address]']
+    @host_port_field = ecs_select[disabled: '[@metadata][host_port]', v1: '[@metadata][input][snmp][host][port]']
+    @host_community_field = ecs_select[disabled: '[@metadata][host_community]', v1: '[@metadata][input][snmp][host][community]']
+
+    # Add the default "host" field to the event, for backwards compatibility, or host.ip in ecs mode
+    unless params.key?('add_field')
+      host_ip_field = ecs_select[disabled: "host", v1: "[host][ip]"]
+      @add_field = { host_ip_field => "%{#{@host_address_field}}" }
+    end
+  end
+
   def register
     validate_oids!
     validate_hosts!
@@ -161,57 +191,82 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
   end
 
   def run(queue)
-    # for now a naive single threaded poller which sleeps for the given interval between
-    # each run. each run polls all the defined hosts for the get and walk options.
-    while !stop?
-      @client_definitions.each do |definition|
-        result = {}
-        if !definition[:get].empty?
-          begin
-            result = result.merge(definition[:client].get(definition[:get], @oid_root_skip, @oid_path_length))
-          rescue => e
-            logger.error("error invoking get operation on #{definition[:host_address]} for OIDs: #{definition[:get]}, ignoring", :exception => e, :backtrace => e.backtrace)
+    # for now a naive single threaded poller which sleeps off the remaining interval between
+    # each run. each run polls all the defined hosts for the get, table and walk options.
+    stoppable_interval_runner.every(@interval, "polling hosts") do
+      poll_clients(queue)
+    end
+  end
+
+  def poll_clients(queue)
+    @client_definitions.each do |definition|
+      client = definition[:client]
+      host = definition[:host_address]
+      result = {}
+
+      if !definition[:get].empty?
+        oids = definition[:get]
+        begin
+          data = client.get(oids, @oid_root_skip, @oid_path_length)
+          if data
+            result.update(data)
+          else
+            logger.debug? && logger.debug("get operation returned no response", host: host, oids: oids)
           end
+        rescue => e
+          logger.error("error invoking get operation, ignoring", host: host, oids: oids, exception: e, backtrace: e.backtrace)
         end
-        if  !definition[:walk].empty?
-          definition[:walk].each do |oid|
-            begin
-              result = result.merge(definition[:client].walk(oid, @oid_root_skip, @oid_path_length))
-            rescue => e
-              logger.error("error invoking walk operation on OID: #{oid}, ignoring", :exception => e, :backtrace => e.backtrace)
+      end
+
+      if !definition[:walk].empty?
+        definition[:walk].each do |oid|
+          begin
+            data = client.walk(oid, @oid_root_skip, @oid_path_length)
+            if data
+              result.update(data)
+            else
+              logger.debug? && logger.debug("walk operation returned no response", host: host, oid: oid)
             end
+          rescue => e
+            logger.error("error invoking walk operation, ignoring", host: host, oid: oid, exception: e, backtrace: e.backtrace)
           end
         end
+      end
 
-        if  !Array(@tables).empty?
-          @tables.each do |table_entry|
-            begin
-              result = result.merge(definition[:client].table(table_entry, @oid_root_skip, @oid_path_length))
-            rescue => e
-              logger.error("error invoking table operation on OID: #{table_entry['name']}, ignoring", :exception => e, :backtrace => e.backtrace)
+      if !Array(@tables).empty?
+        @tables.each do |table_entry|
+          begin
+            data = client.table(table_entry, @oid_root_skip, @oid_path_length)
+            if data
+              result.update(data)
+            else
+              logger.debug? && logger.debug("table operation returned no response", host: host, table: table_entry)
             end
+          rescue => e
+            logger.error("error invoking table operation, ignoring",
+                         host: host, table_name: table_entry['name'], exception: e, backtrace: e.backtrace)
           end
         end
-
-        unless result.empty?
-          metadata = {
-            "host_protocol" => definition[:host_protocol],
-            "host_address" => definition[:host_address],
-            "host_port" => definition[:host_port],
-            "host_community" => definition[:host_community],
-          }
-          result["@metadata"] = metadata
-
-          event = LogStash::Event.new(result)
-          decorate(event)
-          queue << event
-        end
       end
 
-      Stud.stoppable_sleep(@interval) { stop? }
+      unless result.empty?
+        event = targeted_event_factory.new_event(result)
+        event.set(@host_protocol_field, definition[:host_protocol])
+        event.set(@host_address_field, definition[:host_address])
+        event.set(@host_port_field, definition[:host_port])
+        event.set(@host_community_field, definition[:host_community])
+        decorate(event)
+        queue << event
+      else
+        logger.debug? && logger.debug("no snmp data retrieved", host: definition[:host_address])
+      end
     end
   end
 
+  def stoppable_interval_runner
+    StoppableIntervalRunner.new(self)
+  end
+
   def close
     @client_definitions.each do |definition|
       begin
@@ -222,10 +277,13 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
     end
   end
 
+  def stop
+  end
+
   private
 
   OID_REGEX = /^\.?([0-9\.]+)$/
-  HOST_REGEX = /^(?<host_protocol>udp|tcp):(?<host_address>.+)\/(?<host_port>\d+)$/i
+  HOST_REGEX = /^(?<host_protocol>\w+):(?<host_address>.+)\/(?<host_port>\d+)$/i
   VERSION_REGEX =/^1|2c|3$/
 
   def validate_oids!
@@ -295,4 +353,45 @@ class LogStash::Inputs::Snmp < LogStash::Inputs::Base
   def validate_strip!
     raise(LogStash::ConfigurationError, "you can not specify both oid_root_skip and oid_path_length") if @oid_root_skip > 0 and @oid_path_length > 0
   end
+
+  ##
+  # The StoppableIntervalRunner is capable of running a block of code at a
+  # repeating interval, while respecting the stop condition of the plugin.
+  class StoppableIntervalRunner
+    ##
+    # @param plugin [#logger,#stop?]
+    def initialize(plugin)
+      @plugin = plugin
+    end
+
+    ##
+    # Runs the provided block repeatedly using the provided interval.
+    # After executing the block, the remainder of the interval if any is slept off
+    # using an interruptible sleep.
+    # If no time remains, a warning is emitted to the logs.
+    #
+    # @param interval_seconds [Integer,Float]
+    # @param desc [String] (default: "operation"): a description to use when logging
+    # @yield
+    def every(interval_seconds, desc="operation", &block)
+      until @plugin.stop?
+        start_time = Time.now
+
+        yield
+
+        duration_seconds = Time.now - start_time
+        if duration_seconds >= interval_seconds
+          @plugin.logger.warn("#{desc} took longer than the configured interval", :interval_seconds => interval_seconds, :duration_seconds => duration_seconds.round(3))
+        else
+          remaining_interval = interval_seconds - duration_seconds
+          sleep(remaining_interval)
+        end
+      end
+    end
+
+    # @api private
+    def sleep(duration)
+      Stud.stoppable_sleep(duration) { @plugin.stop? }
+    end
+  end
 end

data/logstash-input-snmp.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-snmp'
-  s.version       = '1.2.6'
+  s.version       = '1.3.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = "SNMP input plugin"
   s.description   = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,8 +20,12 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
 end

data/spec/inputs/integration/it_spec.rb

@@ -0,0 +1,203 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/snmp"
+
+describe LogStash::Inputs::Snmp do
+  let(:config) { {"get" => ["1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"]} }
+  let(:plugin) { LogStash::Inputs::Snmp.new(config)}
+
+  shared_examples "snmp plugin return single event" do
+    it "should have OID value" do
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+      event = queue.pop
+
+      expect(event).to be_a(LogStash::Event)
+      expect(event.get("iso.org.dod.internet.mgmt.mib-2.system.sysUpTime.sysUpTimeInstance")).to be_a Integer
+      expect(event.get("iso.org.dod.internet.mgmt.mib-2.system.sysName.0")).to be_a String
+      expect(event.get("iso.org.dod.internet.mgmt.mib-2.system.sysDescr.0")).to be_a String
+    end
+  end
+
+  shared_examples "snmp plugin return one udp event and one tcp event" do |config|
+    it "should have one udp from snmp1 and one tcp from snmp2" do
+      events = input(config) { |_, queue| 2.times.collect { queue.pop } }
+      udp = 0; tcp = 0
+      events.each { |event|
+        if event.get("[@metadata][host_protocol]") == "udp"
+          udp += 1
+          expect(event.get("[@metadata][host_protocol]")).to eq("udp")
+          expect(event.get("[@metadata][host_address]")).to eq("snmp1")
+          expect(event.get("[@metadata][host_port]")).to eq("161")
+        else
+          tcp += 1
+          expect(event.get("[@metadata][host_protocol]")).to eq("tcp")
+          expect(event.get("[@metadata][host_address]")).to eq("snmp2")
+          expect(event.get("[@metadata][host_port]")).to eq("162")
+        end
+      }
+      expect(udp).to eq(1)
+      expect(tcp).to eq(1)
+    end
+  end
+
+  describe "against single snmp server with snmpv2 and udp", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}]})}
+    it_behaves_like "snmp plugin return single event"
+  end
+
+  describe "against single server with snmpv3 and tcp", :integration => true do
+    let(:config) { super().merge({
+       "hosts" => [{"host" => "tcp:snmp1/161", "version" => "3"}],
+       "security_name" => "user_1",
+       "auth_protocol" => "sha",
+       "auth_pass" => "STrP@SSPhr@sE",
+       "priv_protocol" => "aes",
+       "priv_pass" => "STr0ngP@SSWRD161",
+       "security_level" => "authPriv"
+                               })}
+
+    it_behaves_like "snmp plugin return single event"
+  end
+
+  describe "invalid user against snmpv3 server", :integration => true do
+    let(:config) { super().merge({
+                                   "hosts" => [{"host" => "tcp:snmp1/161", "version" => "3"}],
+                                   "security_name" => "user_2",
+                                   "auth_protocol" => "sha",
+                                   "auth_pass" => "STrP@SSPhr@sE",
+                                   "priv_protocol" => "aes",
+                                   "priv_pass" => "STr0ngP@SSWRD161",
+                                   "security_level" => "authPriv"
+                               })}
+
+    it "should have error log" do
+      expect(plugin.logger).to receive(:error).once
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+    end
+  end
+
+  describe "single input plugin on single server with snmpv2 and mix of udp and tcp", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}, {"host" => "tcp:snmp1/161", "community" => "public"}]})}
+    it "should return two events " do
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+
+      host_cnt_snmp1 = queue.select {|event| event.get("host") == "snmp1"}.size
+      expect(queue.size).to eq(2)
+      expect(host_cnt_snmp1).to eq(2)
+    end
+  end
+
+  describe "single input plugin on multiple udp hosts", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}, {"host" => "udp:snmp2/162", "community" => "public"}]})}
+    it "should return two events, one per host" do
+      plugin.register
+      queue = []
+      stop_plugin_after_seconds(plugin)
+      plugin.run(queue)
+      plugin.close
+
+      hosts = queue.map { |event| event.get("host") }.sort
+      expect(queue.size).to eq(2)
+      expect(hosts).to eq(["snmp1", "snmp2"])
+    end
+  end
+
+  describe "multiple pipelines and mix of udp tcp hosts", :integration => true do
+    let(:config) { {"get" => ["1.3.6.1.2.1.1.1.0"], "hosts" => [{"host" => "udp:snmp1/161", "community" => "public"}]} }
+    let(:config2) { {"get" => ["1.3.6.1.2.1.1.1.0"], "hosts" => [{"host" => "tcp:snmp2/162", "community" => "public"}]} }
+    let(:plugin) { LogStash::Inputs::Snmp.new(config)}
+    let(:plugin2) { LogStash::Inputs::Snmp.new(config2)}
+
+    it "should return two events, one per host" do
+      plugin.register
+      plugin2.register
+      queue = []
+      queue2 = []
+      t = Thread.new {
+        stop_plugin_after_seconds(plugin)
+        plugin.run(queue)
+      }
+      t2 = Thread.new {
+        stop_plugin_after_seconds(plugin2)
+        plugin2.run(queue2)
+      }
+      t.join(2100)
+      t2.join(2100)
+      plugin.close
+      plugin2.close
+
+      hosts = [queue.pop, queue2.pop].map { |event| event.get("host") }.sort
+      expect(hosts).to eq(["snmp1", "snmp2"])
+    end
+  end
+
+  describe "multiple plugin inputs and mix of udp tcp hosts", :integration => true do
+    config = <<-CONFIG
+        input {
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "udp:snmp1/161" community => "public"}]
+          }
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "tcp:snmp2/162" community => "public"}]
+          }
+        }
+    CONFIG
+
+    it_behaves_like "snmp plugin return one udp event and one tcp event", config
+  end
+
+  describe "two plugins on different hosts with snmpv3 with same security name with different credentials and mix of udp and tcp", :integration => true do
+    config = <<-CONFIG
+        input {
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "udp:snmp1/161" version => "3"}]
+            security_name => "user_1"
+            auth_protocol => "sha"
+            auth_pass => "STrP@SSPhr@sE"
+            priv_protocol => "aes"
+            priv_pass => "STr0ngP@SSWRD161"
+            security_level => "authPriv"
+          }
+          snmp {
+            get => ["1.3.6.1.2.1.1.1.0"]
+            hosts => [{host => "tcp:snmp2/162" version => "3"}]
+            security_name => "user_1"
+            auth_protocol => "sha"
+            auth_pass => "STrP@SSPhr@sE"
+            priv_protocol => "aes"
+            priv_pass => "STr0ngP@SSWRD162"
+            security_level => "authPriv"
+          }
+        }
+    CONFIG
+
+    it_behaves_like "snmp plugin return one udp event and one tcp event", config
+  end
+
+  describe "single host with tcp over ipv6", :integration => true do
+    let(:config) { super().merge({"hosts" => [{"host" => "tcp:[2001:3984:3989::161]/161"}]})}
+    it_behaves_like "snmp plugin return single event"
+  end
+
+  def stop_plugin_after_seconds(plugin)
+      Thread.new{
+        sleep(2)
+        plugin.do_stop
+      }
+  end
+
+end

data/spec/inputs/snmp_spec.rb

@@ -1,9 +1,11 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/devutils/rspec/shared_examples"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 require "logstash/inputs/snmp"
 
-describe LogStash::Inputs::Snmp do
+describe LogStash::Inputs::Snmp, :ecs_compatibility_support do
+
   let(:mock_client) { double("LogStash::SnmpClient") }
 
   it_behaves_like "an interruptible input plugin" do
@@ -130,45 +132,262 @@ describe LogStash::Inputs::Snmp do
     end
   end
 
-  context "@metadata" do
-    before do
+  ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+    before(:each) do
+      allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
       expect(LogStash::SnmpClient).to receive(:new).and_return(mock_client)
-      expect(mock_client).to receive(:get).and_return({"foo" => "bar"})
       # devutils in v6 calls close on the test pipelines while it does not in v7+
-      expect(mock_client).to receive(:close).at_most(:once)
+      allow(mock_client).to receive(:close).at_most(:once)
     end
 
-    it "shoud add @metadata fields and add default host field" do
-      config = <<-CONFIG
+    context 'mocked get' do
+
+      before do
+        expect(mock_client).to receive(:get).and_return({"foo" => "bar"})
+      end
+
+      it "shoud add @metadata fields and add default host field" do
+        config = <<-CONFIG
           input {
             snmp {
               get => ["1.3.6.1.2.1.1.1.0"]
-              hosts => [{host => "udp:127.0.0.1/161" community => "public"}]
+              hosts => [{ host => "udp:127.0.0.1/161" community => "public" }]
             }
           }
-      CONFIG
-      event = input(config) { |_, queue| queue.pop }
-
-      expect(event.get("[@metadata][host_protocol]")).to eq("udp")
-      expect(event.get("[@metadata][host_address]")).to eq("127.0.0.1")
-      expect(event.get("[@metadata][host_port]")).to eq("161")
-      expect(event.get("[@metadata][host_community]")).to eq("public")
-      expect(event.get("host")).to eq("127.0.0.1")
-    end
+        CONFIG
+        event = input(config) { |_, queue| queue.pop }
+
+        if ecs_select.active_mode == :disabled
+          expect(event.get("[@metadata][host_protocol]")).to eq("udp")
+          expect(event.get("[@metadata][host_address]")).to eq("127.0.0.1")
+          expect(event.get("[@metadata][host_port]")).to eq("161")
+          expect(event.get("[@metadata][host_community]")).to eq("public")
+          expect(event.get("host")).to eql("127.0.0.1")
+        else
+          expect(event.get("[@metadata][input][snmp][host][protocol]")).to eq("udp")
+          expect(event.get("[@metadata][input][snmp][host][address]")).to eq("127.0.0.1")
+          expect(event.get("[@metadata][input][snmp][host][port]")).to eq('161')
+          expect(event.get("[@metadata][input][snmp][host][community]")).to eq("public")
+          expect(event.get("host")).to eql('ip' => "127.0.0.1")
+        end
+      end
 
-    it "shoud add custom host field" do
-      config = <<-CONFIG
+      it "should add custom host field (legacy metadata)" do
+        config = <<-CONFIG
           input {
             snmp {
               get => ["1.3.6.1.2.1.1.1.0"]
-              hosts => [{host => "udp:127.0.0.1/161" community => "public"}]
+              hosts => [{ host => "udp:127.0.0.1/161" community => "public" }]
               add_field => { host => "%{[@metadata][host_protocol]}:%{[@metadata][host_address]}/%{[@metadata][host_port]},%{[@metadata][host_community]}" }
             }
           }
-      CONFIG
-      event = input(config) { |_, queue| queue.pop }
+        CONFIG
+        event = input(config) { |_, queue| queue.pop }
+
+        expect(event.get("host")).to eq("udp:127.0.0.1/161,public")
+      end if ecs_select.active_mode == :disabled
+
+      it "should add custom host field (ECS mode)" do
+        config = <<-CONFIG
+          input {
+            snmp {
+              get => ["1.3.6.1.2.1.1.1.0"]
+              hosts => [{ host => "tcp:192.168.1.11/1161" }]
+              add_field => { "[host][formatted]" => "%{[@metadata][input][snmp][host][protocol]}://%{[@metadata][input][snmp][host][address]}:%{[@metadata][input][snmp][host][port]}" }
+            }
+          }
+        CONFIG
+        event = input(config) { |_, queue| queue.pop }
+
+        expect(event.get("host")).to eq('formatted' => "tcp://192.168.1.11:1161")
+      end if ecs_select.active_mode != :disabled
+
+      it "should target event data" do
+        config = <<-CONFIG
+          input {
+            snmp {
+              get => ["1.3.6.1.2.1.1.1.0"]
+              hosts => [{ host => "udp:127.0.0.1/161" community => "public" }]
+              target => "snmp_data"
+            }
+          }
+        CONFIG
+        event = input(config) { |_, queue| queue.pop }
+
+        expect( event.include?('foo') ).to be false
+        expect( event.get('[snmp_data]') ).to eql 'foo' => 'bar'
+      end
+
+    end
+
+    context 'mocked nil get response' do
+
+      let(:config) do
+        {
+            'get' => ["1.3.6.1.2.1.1.1.0"],
+            "hosts" => [{"host" => "udp:127.0.0.1/161", "community" => "public"}]
+        }
+      end
+
+      let(:logger) { double("Logger").as_null_object }
+
+      before do
+        expect(mock_client).to receive(:get).once.and_return(nil)
+        allow_any_instance_of(described_class).to receive(:logger).and_return(logger)
+        expect(logger).not_to receive(:error)
+      end
+
+      it 'generates no events when client returns no response' do
+        input = described_class.new(config).tap { |input| input.register }
+        input.poll_clients queue = Queue.new
+
+        expect( queue.size ).to eql 0
+      end
+    end
+
+    context 'mocked nil table response' do
+
+      let(:config) do
+        {
+            'tables' => [
+                { 'name' => "a1Table", 'columns' => ["1.3.6.1.4.1.3375.2.2.5.2.3.1.1"] }
+            ],
+            "hosts" => [{"host" => "udp:127.0.0.1/161", "community" => "public"}]
+        }
+      end
+
+      let(:logger) { double("Logger").as_null_object }
+
+      before do
+        expect(mock_client).to receive(:table).once.and_return(nil)
+        allow_any_instance_of(described_class).to receive(:logger).and_return(logger)
+        expect(logger).not_to receive(:error)
+      end
+
+      it 'generates no events when client returns no response' do
+        input = described_class.new(config).tap { |input| input.register }
+        input.poll_clients queue = Queue.new
+
+        expect( queue.size ).to eql 0
+      end
+    end
+
+    context 'mocked nil walk response' do
+
+      let(:config) do
+        {
+            'walk' => ["1.3.6.1.2.1.1"],
+            "hosts" => [{"host" => "udp:127.0.0.1/161", "community" => "public"}]
+        }
+      end
+
+      let(:logger) { double("Logger").as_null_object }
+
+      before do
+        expect(mock_client).to receive(:walk).once.and_return(nil)
+        allow_any_instance_of(described_class).to receive(:logger).and_return(logger)
+        expect(logger).not_to receive(:error)
+      end
+
+      it 'generates no events when client returns no response' do
+        input = described_class.new(config).tap { |input| input.register }
+        input.poll_clients queue = Queue.new
+
+        expect( queue.size ).to eql 0
+      end
+    end
+
+  end
+
+  context "StoppableIntervalRunner" do
+    let(:stop_holder) { Struct.new(:value).new(false) }
+
+    before(:each) do
+      allow(plugin).to receive(:stop?) { stop_holder.value }
+    end
+
+    let(:plugin) do
+      double("Plugin").tap do |dbl|
+        allow(dbl).to receive(:logger).and_return(double("Logger").as_null_object)
+        allow(dbl).to receive(:stop?) { stop_holder.value }
+      end
+    end
+
+    subject(:interval_runner) { LogStash::Inputs::Snmp::StoppableIntervalRunner.new(plugin) }
+
+    context "#every" do
+      context "when the plugin is stopped" do
+        let(:interval_seconds) { 2 }
+        it 'does not yield the block' do
+          stop_holder.value = true
+          expect { |yielder| interval_runner.every(interval_seconds, &yielder) }.to_not yield_control
+        end
+      end
+
+      context "when the yield takes shorter than the interval" do
+        let(:duration_seconds) { 1 }
+        let(:interval_seconds) { 2 }
+
+        it 'sleeps off the remainder' do
+          allow(interval_runner).to receive(:sleep).and_call_original
+
+          interval_runner.every(interval_seconds) do
+            Kernel::sleep(duration_seconds) # non-stoppable
+            stop_holder.value = true # prevent re-runs
+          end
+
+          expect(interval_runner).to have_received(:sleep).with(a_value_within(0.1).of(1))
+        end
+      end
+
+      context "when the yield takes longer than the interval" do
+        let(:duration_seconds) { 2 }
+        let(:interval_seconds) { 1 }
+
+        it 'logs a warning to the plugin' do
+          allow(interval_runner).to receive(:sleep).and_call_original
 
-      expect(event.get("host")).to eq("udp:127.0.0.1/161,public")
+          interval_runner.every(interval_seconds) do
+            Kernel::sleep(duration_seconds) # non-stoppable
+            stop_holder.value = true # prevent re-runs
+          end
+
+          expect(interval_runner).to_not have_received(:sleep)
+
+          expect(plugin.logger).to have_received(:warn).with(a_string_including("took longer"), a_hash_including(:interval_seconds => interval_seconds, :duration_seconds => a_value_within(0.1).of(duration_seconds)))
+        end
+      end
+
+      it 'runs regularly until the plugin is stopped' do
+        timestamps = []
+
+        thread = Thread.new do
+          interval_runner.every(1) do
+            timestamps << Time.now
+            Kernel::sleep(Random::rand(0.8))
+          end
+        end
+
+        Kernel::sleep(5)
+        expect(thread).to be_alive
+
+        stop_holder.value = true
+        Kernel::sleep(1)
+
+        aggregate_failures do
+          expect(thread).to_not be_alive
+          expect(timestamps.count).to be_within(1).of(5)
+
+          timestamps.each_cons(2) do |previous, current|
+            # ensure each start time is very close to 1s after the previous.
+            expect(current - previous).to be_within(0.05).of(1)
+          end
+
+          thread.kill if thread.alive?
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-snmp
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.3.1
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-21 00:00:00.000000000 Z
+date: 2021-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -33,23 +33,45 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.22
-    - - "<"
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
-  name: stud
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-event_support
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.22
-    - - "<"
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -64,6 +86,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -414,6 +456,7 @@ files:
 - logstash-input-snmp.gemspec
 - spec/fixtures/RFC1213-MIB.dic
 - spec/fixtures/collision.dic
+- spec/inputs/integration/it_spec.rb
 - spec/inputs/snmp/base_client_spec.rb
 - spec/inputs/snmp/mib_spec.rb
 - spec/inputs/snmp_spec.rb
@@ -440,14 +483,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: SNMP input plugin
 test_files:
 - spec/fixtures/RFC1213-MIB.dic
 - spec/fixtures/collision.dic
+- spec/inputs/integration/it_spec.rb
 - spec/inputs/snmp/base_client_spec.rb
 - spec/inputs/snmp/mib_spec.rb
 - spec/inputs/snmp_spec.rb