logstash-filter-cidrtagmap 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +26 -2
- data/lib/logstash/filters/cidrtagmap.rb +79 -30
- data/logstash-filter-cidrtagmap.gemspec +5 -3
- metadata +19 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 46643c17c700e7a7d0df67803c025b0cfd5eaddc
|
|
4
|
+
data.tar.gz: 9bf4896f9bb7c02e28ea37100932a5c53367a5f5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 352ede9a9fd1a2044ec928d612c917b56bbafee51cb62b693b9d62aa52c41a050fe2c3fb05376d1530507a3d367077f5ccf203c80ebe5c5a43cfba498b7cd2b2
|
|
7
|
+
data.tar.gz: d30675b270005ca35b78620117db7958733297319d3da9ee392f548ecf24e674054691838ed80b3c5d83013baf4a5f25cd51743240eeaf8e31f71769eb8e312a
|
data/README.md
CHANGED
|
@@ -2,9 +2,12 @@ This logstash filter tags events according to a list of CIDR to tag mappings, an
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
Example:
|
|
5
|
+
--------
|
|
5
6
|
|
|
6
7
|
```
|
|
7
8
|
cidrtagmap {
|
|
9
|
+
#redisserver => '127.0.0.1'
|
|
10
|
+
#redisnamespace => "ereiamjh"
|
|
8
11
|
mapfilepath => "/path/to/ipmap/file"
|
|
9
12
|
asnmapfilepath => "/path/to/asnmap/file"
|
|
10
13
|
ipfieldlist => [
|
|
@@ -19,7 +22,15 @@ cidrtagmap {
|
|
|
19
22
|
}
|
|
20
23
|
```
|
|
21
24
|
|
|
22
|
-
|
|
25
|
+
|
|
26
|
+
CIDR map configuration
|
|
27
|
+
----------------------
|
|
28
|
+
|
|
29
|
+
You must specify a map source. Currently there are two forms of this: file based and redis based.
|
|
30
|
+
|
|
31
|
+
### File based configuration:
|
|
32
|
+
|
|
33
|
+
* mapfilepath points to an external / stand alone text file consisting of lines of the form:
|
|
23
34
|
|
|
24
35
|
```
|
|
25
36
|
<network>/<mask>,<tag>
|
|
@@ -33,15 +44,28 @@ touch <mapfilepath>.RELOAD
|
|
|
33
44
|
```
|
|
34
45
|
|
|
35
46
|
|
|
36
|
-
|
|
47
|
+
### Redis based configuration:
|
|
48
|
+
|
|
49
|
+
* redisserver = name or ip address of redis server to connect to. If you define this you should also define:
|
|
50
|
+
* redisnamespace = string that will act as a prefix to variables stored in redis
|
|
37
51
|
|
|
38
52
|
|
|
53
|
+
In redis then you should define two items:
|
|
54
|
+
* redisnamespace.cidrmap = a hash with cidr => tag kv pairs
|
|
55
|
+
* redisnamespace.reloadmap = 1|0 - tell filter to reload map
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
Other configuration:
|
|
59
|
+
--------------------
|
|
60
|
+
|
|
39
61
|
* ipfieldlist (required) is a list of event fields that will be eligible for mapping. Everything that matches
|
|
40
62
|
will be put in a structure subtending an item called cidrtagmap, so
|
|
41
63
|
from the above example a match of the [netflow][dst_address] field would add
|
|
42
64
|
cidrtagmap.netflow.dst_address.tag. A pair to this field will be cidrtagmap.netflow.dst_address.match
|
|
43
65
|
which indicates which rule was matched for the mapping.
|
|
44
66
|
|
|
67
|
+
* asnmapfilepath (optional) points to a copy of this file: ftp://ftp.arin.net/info/asn.txt
|
|
68
|
+
|
|
45
69
|
* asnfieldlist (optional) is a list of fields presumed to contain asn numbers. Everything that matches
|
|
46
70
|
will add e.g. cidrtagmap.netflow.dst_as.asname
|
|
47
71
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
require "logstash/filters/base"
|
|
3
3
|
require "logstash/namespace"
|
|
4
4
|
require 'ipaddr'
|
|
5
|
+
require 'redis'
|
|
5
6
|
|
|
6
7
|
class MapEntry
|
|
7
8
|
attr_reader :range,:tag
|
|
@@ -28,14 +29,29 @@ class LogStash::Filters::CIDRTagMap < LogStash::Filters::Base
|
|
|
28
29
|
|
|
29
30
|
config :mapfilepath, :validate => :string, :default => 'cidrmap.txt'
|
|
30
31
|
config :asnmapfilepath, :validate => :string, :default => 'asn.txt'
|
|
31
|
-
config :ipfieldlist, :
|
|
32
|
-
config :
|
|
32
|
+
config :ipfieldlist, :optional=> true, :list => true , :validate => :string
|
|
33
|
+
config :redisserver, :optional=> true, :validate => :string
|
|
34
|
+
config :redisnamespace, :optional=> true, :validate => :string
|
|
35
|
+
config :asnfieldlist, :list => true, :validate => :string
|
|
33
36
|
|
|
34
37
|
|
|
35
38
|
private
|
|
36
39
|
|
|
40
|
+
|
|
41
|
+
def loadAsnMap
|
|
42
|
+
begin
|
|
43
|
+
asntable = File.readlines(@asnmapfilepath)
|
|
44
|
+
regex = /^ (\d+?)\s+(.+?)\s+/
|
|
45
|
+
@asnmap = Hash[asntable.collect { |line| line.match(regex)}.select {|each| not each.nil?}.collect{|each| [each[1],each[2]] }]
|
|
46
|
+
@logger.info("cidrtagmap: loaded asn map file #{@asnmapfilepath}")
|
|
47
|
+
rescue Exception => e
|
|
48
|
+
@logger.warn("cidrtagmap: error loading asn map file #{@asnmapfilepath}")
|
|
49
|
+
@logger.warn("cidrtagmap: #{e.inspect}")
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
37
53
|
def loadLatestMap
|
|
38
|
-
if File.exist?(@reloadFlagPath) or @cidrMap.nil?
|
|
54
|
+
if (not @redisserver) and (File.exist?(@reloadFlagPath) or @cidrMap.nil?)
|
|
39
55
|
@logger.debug("cidrtagmap: need to load, getting mutex")
|
|
40
56
|
@mutex.synchronize {
|
|
41
57
|
# Test again now that we have the floor just in case someone else did it already
|
|
@@ -70,18 +86,38 @@ class LogStash::Filters::CIDRTagMap < LogStash::Filters::Base
|
|
|
70
86
|
@logger.warn("cidrtagmap: error opening map file #{@mapfilepath}\n")
|
|
71
87
|
@mapFile = nil
|
|
72
88
|
end
|
|
73
|
-
|
|
74
|
-
asntable = File.readlines(@asnmapfilepath)
|
|
75
|
-
regex = /^ (\d+?)\s+(.+?)\s+/
|
|
76
|
-
@asnmap = Hash[asntable.collect { |line| line.match(regex)}.select {|each| not each.nil?}.collect{|each| [each[1],each[2]] }]
|
|
77
|
-
rescue Exception => e
|
|
78
|
-
@logger.warn("cidrtagmap: error loading asn map file #{@asnmapfilepath}\n")
|
|
79
|
-
@logger.warn("cidrtagmap: #{e.inspect}")
|
|
80
|
-
end
|
|
89
|
+
loadAsnMap
|
|
81
90
|
else
|
|
82
91
|
@logger.debug("cidrtagmap: someone already loaded the map - I'm outta here")
|
|
83
92
|
end
|
|
84
93
|
}
|
|
94
|
+
elsif @redisserver
|
|
95
|
+
if not @redisnamespace
|
|
96
|
+
@logger.warn("cidrtagmap: redisnamespace not defined - using cidrtagmap")
|
|
97
|
+
@redisnamespace = 'cidrtagmap'
|
|
98
|
+
end
|
|
99
|
+
if @redis["#{@redisnamespace}.reloadmap"] == '1' or @cidrMap.nil?
|
|
100
|
+
@mutex.synchronize {
|
|
101
|
+
if @redis["#{@redisnamespace}.reloadmap"] == '1' or @cidrMap.nil?
|
|
102
|
+
@redis["#{@redisnamespace}.reloadmap"] = '0'
|
|
103
|
+
@logger.info("cidrtagmap: refreshing map from redis server at #{@redisserver} using namespace '#{@redisnamespace}'")
|
|
104
|
+
begin
|
|
105
|
+
rawcidrmap = @redis.hgetall("#{@redisnamespace}.cidrmap")
|
|
106
|
+
@cidrMap = rawcidrmap.each.map{ |cidr,tag|
|
|
107
|
+
MapEntry.new("#{cidr},#{tag}")
|
|
108
|
+
}
|
|
109
|
+
@cidrMap = @cidrMap.reject { |item| item.nil? }
|
|
110
|
+
@logger.info("cidrtagmap: loaded #{@cidrMap.inspect}")
|
|
111
|
+
rescue Exception => e
|
|
112
|
+
@logger.error("cidrtagmap: error attempting to load map from redis")
|
|
113
|
+
@logger.error(e.inspect)
|
|
114
|
+
end
|
|
115
|
+
loadAsnMap
|
|
116
|
+
else
|
|
117
|
+
@logger.debug("cidrtagmap: someone already loaded the map - I'm outta here")
|
|
118
|
+
end
|
|
119
|
+
}
|
|
120
|
+
end
|
|
85
121
|
end
|
|
86
122
|
end
|
|
87
123
|
|
|
@@ -114,33 +150,46 @@ class LogStash::Filters::CIDRTagMap < LogStash::Filters::Base
|
|
|
114
150
|
public
|
|
115
151
|
def register
|
|
116
152
|
@mutex = Mutex.new
|
|
117
|
-
|
|
118
|
-
|
|
153
|
+
if @redisserver
|
|
154
|
+
begin
|
|
155
|
+
@redis = Redis.new(:host=>@redisserver)
|
|
156
|
+
@logger.info("cidrtagmap: connected to redis server at #{@redisserver}")
|
|
157
|
+
rescue Exception => e
|
|
158
|
+
@logger.error("cidrtagmap: failed to connect to redis server at #{@redisserver}")
|
|
159
|
+
end
|
|
160
|
+
else
|
|
161
|
+
@reloadFlagPath = "#{@mapfilepath}.RELOAD"
|
|
162
|
+
@logger.info("cidrtagmap: NOTE: touch #{@reloadFlagPath} to force map reload")
|
|
163
|
+
end
|
|
119
164
|
loadLatestMap
|
|
120
165
|
end
|
|
121
166
|
|
|
122
167
|
public
|
|
123
168
|
def filter(event)
|
|
124
169
|
return unless filter?(event)
|
|
125
|
-
|
|
126
|
-
@ipfieldlist
|
|
127
|
-
@
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
170
|
+
loadLatestMap
|
|
171
|
+
if @ipfieldlist
|
|
172
|
+
@ipfieldlist.each { |fieldname|
|
|
173
|
+
@logger.debug("cidrtagmap: looking for ipfield '#{fieldname}'")
|
|
174
|
+
if ipvalue = event.get(fieldname)
|
|
175
|
+
@logger.debug("cidrtagmap: I found ipfield #{fieldname} with value #{ipvalue}")
|
|
176
|
+
mapping = mapForIp(ipvalue)
|
|
177
|
+
if mapping
|
|
178
|
+
@logger.debug("cidrtagmap: I mapped IP address #{ipvalue} to #{mapping.tag} via range #{mapping.range.to_s}")
|
|
179
|
+
event.set("[cidrtagmap]#{fieldname}[tag]",mapping.tag)
|
|
180
|
+
event.set("[cidrtagmap]#{fieldname}[match]",mapping.range.to_s)
|
|
181
|
+
filter_matched(event)
|
|
182
|
+
end
|
|
136
183
|
end
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
184
|
+
}
|
|
185
|
+
else
|
|
186
|
+
@logger.warn("cidrtagmap: No IP field list defined - not attempting to translate ip addresses!")
|
|
187
|
+
end
|
|
188
|
+
if @asnfieldlist
|
|
189
|
+
@asnfieldlist.each { |fieldname|
|
|
190
|
+
@logger.debug("cidrtagmap: looking for asnfield '#{fieldname}'")
|
|
142
191
|
if asvalue = event.get(fieldname)
|
|
143
|
-
@logger.debug("cidrtagmap: I found
|
|
192
|
+
@logger.debug("cidrtagmap: I found asnfield #{fieldname} with value #{asvalue}")
|
|
144
193
|
asname = asNameForNumber(asvalue)
|
|
145
194
|
if asname
|
|
146
195
|
@logger.debug("cidrtagmap: I mapped as number #{asvalue} to #{asname}")
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-filter-cidrtagmap'
|
|
3
|
-
s.version = '2.
|
|
3
|
+
s.version = '2.1.0'
|
|
4
4
|
s.licenses = ['Apache-2.0']
|
|
5
|
-
s.summary = "Filter adds tags to
|
|
6
|
-
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
|
5
|
+
s.summary = "Filter adds tags to events in logstash based on a table of cidr->name mappings and optionally adds asn name fields"
|
|
6
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program. Filter adds tags to events in logstash based on a table of cidr->name mappings and optionally adds asn name fields"
|
|
7
7
|
s.authors = ["svdasein"]
|
|
8
8
|
s.email = 'daveparker01@gmail.com'
|
|
9
9
|
s.homepage = "https://github.com/svdasein/cidrtagmap"
|
|
@@ -20,4 +20,6 @@ Gem::Specification.new do |s|
|
|
|
20
20
|
# Gem dependencies
|
|
21
21
|
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
|
22
22
|
s.add_development_dependency 'logstash-devutils'
|
|
23
|
+
|
|
24
|
+
s.add_runtime_dependency "redis","~> 3.0"
|
|
23
25
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-filter-cidrtagmap
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- svdasein
|
|
@@ -38,9 +38,24 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: redis
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '3.0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '3.0'
|
|
41
55
|
description: This gem is a Logstash plugin required to be installed on top of the
|
|
42
56
|
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
|
43
|
-
gem is not a stand-alone program
|
|
57
|
+
gem is not a stand-alone program. Filter adds tags to events in logstash based
|
|
58
|
+
on a table of cidr->name mappings and optionally adds asn name fields
|
|
44
59
|
email: daveparker01@gmail.com
|
|
45
60
|
executables: []
|
|
46
61
|
extensions: []
|
|
@@ -80,8 +95,8 @@ rubyforge_project:
|
|
|
80
95
|
rubygems_version: 2.5.1
|
|
81
96
|
signing_key:
|
|
82
97
|
specification_version: 4
|
|
83
|
-
summary: Filter adds tags to
|
|
84
|
-
|
|
98
|
+
summary: Filter adds tags to events in logstash based on a table of cidr->name mappings and
|
|
99
|
+
optionally adds asn name fields
|
|
85
100
|
test_files:
|
|
86
101
|
- spec/filters/example_spec.rb
|
|
87
102
|
- spec/spec_helper.rb
|