logstash-filter-cidrtagmap 1.2.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a84d59293902f5d951c56c730f818b102505a79
-  data.tar.gz: 8386370459f96c62abb44299c286e50099842b22
+  metadata.gz: edb209f7d360805f5be547d28728b63e180f3a89
+  data.tar.gz: 3e67480b82b0747e954f43d7ff0891b9db9ab3d6
 SHA512:
-  metadata.gz: 3bd56f6e098c2d190d66e5ae6c3e2c87fce7ac4fe34aefaf5ff59ec6a21698abe16094c218657e25fa681bb9658cab2c8f74ccb31318fdd45ca469e1556a675d
-  data.tar.gz: 755b109660dea605cfaf2c18704a819afa1f4f61db63017f8cfd169653393d40976376df5e55cff5a30fc1e2857e6b3621eade55c9ae91012082fa34d1fb5778
+  metadata.gz: 9025ea9cce3c73b21487e8d4ae7278e1fefe300684e960fb555b64d9372a547b2d37333edae89bad3a88ab0d15287cd2e5007478ffac1f4f6942204e51475ccf
+  data.tar.gz: 3623e9fe0dc6f99ec37b23ea5ea5cde0ca5cbc51b6e2d9785590b9d20de23b2ef5824804a8c6d6f9e2921dbc92c80f5d50e604854a11fee43e80136a1d8ca896

data/README.md

@@ -1,38 +1,49 @@
-This logstash filter tags netflow records according to a list of CIDR to tag mappings, and adds ASN names for the src_as and dst_as fields.
+This logstash filter tags events according to a list of CIDR to tag mappings, and optionally maps ASN numbers to names
 
-The list is an external / stand alone text file consisting of lines of the form:
+
+Example:
+
+```
+cidrtagmap {
+        mapfilepath => "/path/to/ipmap/file"
+        asnmapfilepath => "/path/to/asnmap/file"
+        ipfieldlist => [
+		'host',
+		'[netflow][dst_address]',
+		'[etc]'
+	]
+        asfieldlist => [
+		'[netflow][dst_as]',
+		'[netflow][src_as]
+	]
+}
+```
+
+* mapfilepath (required) points to an  external / stand alone text file consisting of lines of the form:
 
 ```
 <network>/<mask>,<tag>
 ```
 
 The filter can be made to re-load its in-memory representation of the contents of the
-map file without interrupting/restarting the logstash instance by touching a flag file.
+ipmap file without interrupting/restarting the logstash instance by touching a flag file.
 
-When a netflow event matches the CIDR spec, two tags are set:
-
-src_tag = the tag associated with the spec that matched
+```
+touch <mapfilepath>.RELOAD
+```
 
-src_tagMatch = the CIDR spec that matched (as rendered by IPAddr.to_s)
 
-Download the asn map file here: ftp://ftp.arin.net/info/asn.txt 
+* asnmapfilepath (optional) points to a copy of this file: ftp://ftp.arin.net/info/asn.txt 
 
 
-Configuration:
+* ipfieldlist (required) is a list of event fields that will be eligible for mapping.  Everything that matches
+will be put in a structure subtending an item called cidrtagmap, so
+from the above example a match of the [netflow][dst_address] field would add
+cidrtagmap.netflow.dst_address.tag.  A pair to this field will be cidrtagmap.netflow.dst_address.match 
+which indicates which rule was matched for the mapping.
 
-```
-filter{
-        cidrtagmap {
-                mapfilepath => "cidrmap.txt"
-		asnmapfilepath => "asn.txt"
-        }
-}
-```
+* asnfieldlist (optional) is a list of fields presumed to contain asn numbers.  Everything that matches
+will add e.g. cidrtagmap.netflow.dst_as.asname
 
-Tell the filter to reload its maps
 
-```
-touch <mapfilepath>.RELOAD
-```
 
-Reloading is thread safe.

data/lib/logstash/filters/cidrtagmap.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
 require 'ipaddr'
@@ -25,10 +26,10 @@ class LogStash::Filters::CIDRTagMap < LogStash::Filters::Base
 
 	config_name "cidrtagmap"
 
-	milestone 1
-
 	config :mapfilepath, :validate => :string, :default => 'cidrmap.txt'
 	config :asnmapfilepath, :validate => :string, :default => 'asn.txt'
+	config :ipfieldlist, :required => true, :list => true , :validate => :string
+	config :asfieldlist, :list => true, :validate => :string
 
 
 	private
@@ -121,47 +122,36 @@ class LogStash::Filters::CIDRTagMap < LogStash::Filters::Base
 	public
 	def filter(event)
 		return unless filter?(event)
-		if event['netflow']
-			loadLatestMap
-			netflow = event['netflow']
-			if netflow["ipv4_src_addr"]
-				@logger.debug("cidrtagmap: checking for src #{netflow['ipv4_src_addr']}")
-				src_map = mapForIp(netflow["ipv4_src_addr"])
-				if src_map
-					@logger.debug("cidrtagmap: tagging src #{netflow['ipv4_src_addr']} with #{src_map.tag}")
-					netflow["src_tag"] = src_map.tag
-					netflow['src_tagMatch'] = src_map.range.to_s
-					filter_matched(event)
-				end
-			end
-			if netflow["ipv4_dst_addr"]
-				@logger.debug("cidrtagmap: checking for dst #{netflow['ipv4_dst_addr']}")
-				dst_map = mapForIp(netflow["ipv4_dst_addr"])
-				if dst_map
-					@logger.debug("cidrtagmap: tagging dst #{netflow['ipv4_dst_addr']} with #{dst_map.tag}")
-					netflow["dst_tag"] = dst_map.tag
-					netflow["dst_tagMatch"] = dst_map.range.to_s
-					filter_matched(event)
-				end
-			end
-			if netflow["dst_as"]
-				@logger.debug("cidrtagmap: checking for dst_as #{netflow['dst_as']}")
-				dst_asname = asNameForNumber(netflow["dst_as"])
-				if dst_asname
-					@logger.debug("cidrtagmap: tagging dst_as #{netflow['dst_as']} with #{dst_asname}")
-					netflow["dst_as_name"] = dst_asname
+		# There *will* be an @ipfieldlist - this is enforced by the :required directive above
+		@ipfieldlist.each { |fieldname|
+			@logger.debug("cidrtagmap: looking for ipfield '#{fieldname}'")
+			if ipvalue = event.get(fieldname)
+				@logger.debug("cidrtagmap: I found ipfield #{fieldname} with value #{ipvalue}")
+				mapping = mapForIp(ipvalue)
+				if mapping
+					@logger.debug("cidrtagmap: I mapped IP address #{ipvalue} to #{mapping.tag} via range #{mapping.range.to_s}")
+					event.set("[cidrtagmap]#{fieldname}[tag]",mapping.tag)
+					event.set("[cidrtagmap]#{fieldname}[match]",mapping.range.to_s)
 					filter_matched(event)
 				end
 			end
-			if netflow["src_as"]
-				@logger.debug("cidrtagmap: checking for src_as #{netflow['src_as']}")
-				src_asname = asNameForNumber(netflow["src_as"])
-				if src_asname
-					@logger.debug("cidrtagmap: tagging src_as #{netflow['src_as']} with #{src_asname}")
-					netflow["src_as_name"] = src_asname
-					filter_matched(event)
+		}
+		if @asfieldlist
+			@asfieldlist.each { |fieldname|
+				@logger.debug("cidrtagmap: looking for asfield '#{fieldname}'")
+				if asvalue = event.get(fieldname)
+					@logger.debug("cidrtagmap: I found asfield #{fieldname} with value #{asvalue}")
+					asname = asNameForNumber(asvalue)
+					if asname
+						@logger.debug("cidrtagmap: I mapped as number #{asvalue} to #{asname}")
+						event.set("[cidrtagmap]#{fieldname}[asname]",asname)
+						filter_matched(event)
+					end
 				end
-			end
+			}
+		else
+			@logger.debug("cidrtagmap: No as field list defined - not attempting to translate asnames!")
 		end
+
 	end
 end 

data/logstash-filter-cidrtagmap.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-cidrtagmap'
-  s.version         = '1.2.0'
-  s.licenses = ['Apache License (2.0)']
+  s.version         = '2.0.0'
+  s.licenses = ['Apache-2.0']
   s.summary = "Filter adds tags to netflow records in logstash based on a static table of cidr->name and adds asn name fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors = ["svdasein"]
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.8"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_development_dependency 'logstash-devutils'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-cidrtagmap
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 2.0.0
 platform: ruby
 authors:
 - svdasein
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-30 00:00:00.000000000 Z
+date: 2017-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
@@ -57,7 +57,7 @@ files:
 - spec/spec_helper.rb
 homepage: https://github.com/svdasein/cidrtagmap
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
@@ -77,7 +77,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Filter adds tags to netflow records in logstash based on a static table of