logstash-filter-geoip 4.0.4-java → 4.1.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/docs/index.asciidoc +180 -0
- data/lib/logstash-filter-geoip_jars.rb +5 -4
- data/lib/logstash/filters/geoip.rb +20 -137
- data/lib/logstash/filters/geoip/patch.rb +17 -0
- data/logstash-filter-geoip.gemspec +4 -10
- data/spec/filters/geoip_spec.rb +2 -10
- data/vendor/GeoLite2-City.mmdb +0 -0
- data/vendor/jar-dependencies/com/maxmind/db/maxmind-db/1.2.1/maxmind-db-1.2.1.jar +0 -0
- data/vendor/jar-dependencies/com/maxmind/geoip2/geoip2/2.8.0/geoip2-2.8.0.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/filters/logstash-filter-geoip/4.1.0/logstash-filter-geoip-4.1.0.jar +0 -0
- metadata +11 -30
- data/lib/com/fasterxml/jackson/core/jackson-annotations/2.6.0/jackson-annotations-2.6.0.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-core/2.6.4/jackson-core-2.6.4.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-databind/2.6.4/jackson-databind-2.6.4.jar +0 -0
- data/lib/com/maxmind/db/maxmind-db/1.1.0/maxmind-db-1.1.0.jar +0 -0
- data/lib/com/maxmind/geoip2/geoip2/2.5.0/geoip2-2.5.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/geoip2-2.5.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-annotations-2.6.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-core-2.6.4.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-databind-2.6.4.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/maxmind-db-1.1.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c9e2ae6154f3fb6786003715d49ffb91f35a2e34
|
4
|
+
data.tar.gz: 50c92eba1af277a4ddce9f038142a035063b0b07
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 731ce064ee7eae5cb286d12d7736c162016b5fe68c1e6c48697f1ffc930893f9a941cdcd4ecc715fed6077d82ace86c1d8c8446447c7599e8e16a83e55916b32
|
7
|
+
data.tar.gz: a374eeef224c9e60a13d4aac58a89c19d4c47c51fafb60915e8bedcd487221802ddc0c6ec63576e3ed4cf053f390a5b761fd8801d55e1b7b2c7a057fdaae6db1
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
## 4.1.1
|
2
|
+
- Add support for commercial databases from Maxmind.
|
3
|
+
- Add ASN data support via GeoIP2-ISP database.
|
4
|
+
|
5
|
+
## 4.1.0
|
6
|
+
- Removed from RubyGems.org since it was missing the default GeoIP2 database.
|
7
|
+
|
8
|
+
## 4.0.6
|
9
|
+
- Docs: Remove patch classes from the main plugin file
|
10
|
+
- Update of the GeoIP2 DB
|
11
|
+
|
12
|
+
## 4.0.5
|
13
|
+
- Docs: Clarify GeoLite2 database support
|
14
|
+
|
1
15
|
## 4.0.4
|
2
16
|
- Update of the GeoIP2 DB
|
3
17
|
- Target should be merged and not completely overwritten (#98)
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,180 @@
|
|
1
|
+
:plugin: geoip
|
2
|
+
:type: filter
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Geoip
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
The GeoIP filter adds information about the geographical location of IP addresses,
|
24
|
+
based on data from the Maxmind GeoLite2 database. Commercial databases from Maxmind are
|
25
|
+
also supported in this plugin.
|
26
|
+
|
27
|
+
A `[geoip][location]` field is created if
|
28
|
+
the GeoIP lookup returns a latitude and longitude. The field is stored in
|
29
|
+
http://geojson.org/geojson-spec.html[GeoJSON] format. Additionally,
|
30
|
+
the default Elasticsearch template provided with the
|
31
|
+
<<plugins-outputs-elasticsearch,`elasticsearch` output>> maps
|
32
|
+
the `[geoip][location]` field to an http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-geo-point-type.html#_mapping_options[Elasticsearch geo_point].
|
33
|
+
|
34
|
+
As this field is a `geo_point` _and_ it is still valid GeoJSON, you get
|
35
|
+
the awesomeness of Elasticsearch's geospatial query, facet and filter functions
|
36
|
+
and the flexibility of having GeoJSON for all other applications (like Kibana's
|
37
|
+
map visualization).
|
38
|
+
|
39
|
+
[NOTE]
|
40
|
+
--
|
41
|
+
This product includes GeoLite2 data created by MaxMind, available from
|
42
|
+
http://www.maxmind.com. This database is licensed under
|
43
|
+
http://creativecommons.org/licenses/by-sa/4.0/[Creative Commons Attribution-ShareAlike 4.0 International License].
|
44
|
+
|
45
|
+
Versions 4.0.0 and later of the GeoIP filter use the MaxMind GeoLite2 database
|
46
|
+
and support both IPv4 and IPv6 lookups. Versions prior to 4.0.0 use the legacy
|
47
|
+
MaxMind GeoLite database and support IPv4 lookups only.
|
48
|
+
--
|
49
|
+
|
50
|
+
[id="plugins-{type}s-{plugin}-options"]
|
51
|
+
==== Geoip Filter Configuration Options
|
52
|
+
|
53
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
|
54
|
+
|
55
|
+
[cols="<,<,<",options="header",]
|
56
|
+
|=======================================================================
|
57
|
+
|Setting |Input type|Required
|
58
|
+
| <<plugins-{type}s-{plugin}-cache_size>> |<<number,number>>|No
|
59
|
+
| <<plugins-{type}s-{plugin}-database>> |a valid filesystem path|No
|
60
|
+
| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
|
61
|
+
| <<plugins-{type}s-{plugin}-source>> |<<string,string>>|Yes
|
62
|
+
| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
|
63
|
+
| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
|
64
|
+
|=======================================================================
|
65
|
+
|
66
|
+
Also see <<plugins-{type}s-common-options>> for a list of options supported by all
|
67
|
+
filter plugins.
|
68
|
+
|
69
|
+
|
70
|
+
|
71
|
+
[id="plugins-{type}s-{plugin}-cache_size"]
|
72
|
+
===== `cache_size`
|
73
|
+
|
74
|
+
* Value type is <<number,number>>
|
75
|
+
* Default value is `1000`
|
76
|
+
|
77
|
+
GeoIP lookup is surprisingly expensive. This filter uses an cache to take advantage of the fact that
|
78
|
+
IPs agents are often found adjacent to one another in log files and rarely have a random distribution.
|
79
|
+
The higher you set this the more likely an item is to be in the cache and the faster this filter will run.
|
80
|
+
However, if you set this too high you can use more memory than desired.
|
81
|
+
Since the Geoip API upgraded to v2, there is not any eviction policy so far, if cache is full, no more record can be added.
|
82
|
+
Experiment with different values for this option to find the best performance for your dataset.
|
83
|
+
|
84
|
+
This MUST be set to a value > 0. There is really no reason to not want this behavior, the overhead is minimal
|
85
|
+
and the speed gains are large.
|
86
|
+
|
87
|
+
It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter
|
88
|
+
of the same geoip_type share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit
|
89
|
+
to having multiple caches for different instances at different points in the pipeline, that would just increase the
|
90
|
+
number of cache misses and waste memory.
|
91
|
+
|
92
|
+
[id="plugins-{type}s-{plugin}-database"]
|
93
|
+
===== `database`
|
94
|
+
|
95
|
+
* Value type is <<path,path>>
|
96
|
+
* There is no default value for this setting.
|
97
|
+
|
98
|
+
The path to Maxmind's database file that Logstash should use. The default database is GeoLite2-City.
|
99
|
+
GeoLite2-City, GeoLite2-Country, GeoLite2-ASN are the free databases from Maxmind that are supported.
|
100
|
+
GeoIP2-City, GeoIP2-ISP, GeoIP2-Country are the commercial databases from Maxmind that are supported.
|
101
|
+
|
102
|
+
If not specified, this will default to the GeoLite2 City database that ships
|
103
|
+
with Logstash.
|
104
|
+
|
105
|
+
[id="plugins-{type}s-{plugin}-fields"]
|
106
|
+
===== `fields`
|
107
|
+
|
108
|
+
* Value type is <<array,array>>
|
109
|
+
* There is no default value for this setting.
|
110
|
+
|
111
|
+
An array of geoip fields to be included in the event.
|
112
|
+
|
113
|
+
Possible fields depend on the database type. By default, all geoip fields
|
114
|
+
are included in the event.
|
115
|
+
|
116
|
+
For the built-in GeoLite2 City database, the following are available:
|
117
|
+
`city_name`, `continent_code`, `country_code2`, `country_code3`, `country_name`,
|
118
|
+
`dma_code`, `ip`, `latitude`, `longitude`, `postal_code`, `region_name` and `timezone`.
|
119
|
+
|
120
|
+
[id="plugins-{type}s-{plugin}-lru_cache_size"]
|
121
|
+
===== `lru_cache_size` (DEPRECATED)
|
122
|
+
|
123
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
124
|
+
* Value type is <<number,number>>
|
125
|
+
* Default value is `1000`
|
126
|
+
|
127
|
+
GeoIP lookup is surprisingly expensive. This filter uses an LRU cache to take advantage of the fact that
|
128
|
+
IPs agents are often found adjacent to one another in log files and rarely have a random distribution.
|
129
|
+
The higher you set this the more likely an item is to be in the cache and the faster this filter will run.
|
130
|
+
However, if you set this too high you can use more memory than desired.
|
131
|
+
|
132
|
+
Experiment with different values for this option to find the best performance for your dataset.
|
133
|
+
|
134
|
+
This MUST be set to a value > 0. There is really no reason to not want this behavior, the overhead is minimal
|
135
|
+
and the speed gains are large.
|
136
|
+
|
137
|
+
It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter
|
138
|
+
of the same geoip_type share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit
|
139
|
+
to having multiple caches for different instances at different points in the pipeline, that would just increase the
|
140
|
+
number of cache misses and waste memory.
|
141
|
+
|
142
|
+
[id="plugins-{type}s-{plugin}-source"]
|
143
|
+
===== `source`
|
144
|
+
|
145
|
+
* This is a required setting.
|
146
|
+
* Value type is <<string,string>>
|
147
|
+
* There is no default value for this setting.
|
148
|
+
|
149
|
+
The field containing the IP address or hostname to map via geoip. If
|
150
|
+
this field is an array, only the first value will be used.
|
151
|
+
|
152
|
+
[id="plugins-{type}s-{plugin}-tag_on_failure"]
|
153
|
+
===== `tag_on_failure`
|
154
|
+
|
155
|
+
* Value type is <<array,array>>
|
156
|
+
* Default value is `["_geoip_lookup_failure"]`
|
157
|
+
|
158
|
+
Tags the event on failure to look up geo information. This can be used in later analysis.
|
159
|
+
|
160
|
+
[id="plugins-{type}s-{plugin}-target"]
|
161
|
+
===== `target`
|
162
|
+
|
163
|
+
* Value type is <<string,string>>
|
164
|
+
* Default value is `"geoip"`
|
165
|
+
|
166
|
+
Specify the field into which Logstash should store the geoip data.
|
167
|
+
This can be useful, for example, if you have `src_ip` and `dst_ip` fields and
|
168
|
+
would like the GeoIP information of both IPs.
|
169
|
+
|
170
|
+
If you save the data to a target field other than `geoip` and want to use the
|
171
|
+
`geo_point` related functions in Elasticsearch, you need to alter the template
|
172
|
+
provided with the Elasticsearch output and configure the output to use the
|
173
|
+
new template.
|
174
|
+
|
175
|
+
Even if you don't use the `geo_point` mapping, the `[target][location]` field
|
176
|
+
is still valid GeoJSON.
|
177
|
+
|
178
|
+
|
179
|
+
|
180
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -1,5 +1,6 @@
|
|
1
|
-
#
|
2
|
-
require 'logstash/environment'
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
|
3
2
|
|
4
|
-
|
5
|
-
|
3
|
+
require 'jar_dependencies'
|
4
|
+
require_jar('com.maxmind.geoip2', 'geoip2', '2.8.0')
|
5
|
+
require_jar('com.maxmind.db', 'maxmind-db', '1.2.1')
|
6
|
+
require_jar('org.logstash.filters', 'logstash-filter-geoip', '4.1.0')
|
@@ -4,33 +4,6 @@ require "logstash/namespace"
|
|
4
4
|
|
5
5
|
require "logstash-filter-geoip_jars"
|
6
6
|
|
7
|
-
java_import "java.net.InetAddress"
|
8
|
-
java_import "com.maxmind.geoip2.DatabaseReader"
|
9
|
-
java_import "com.maxmind.geoip2.model.CityResponse"
|
10
|
-
java_import "com.maxmind.geoip2.record.Country"
|
11
|
-
java_import "com.maxmind.geoip2.record.Subdivision"
|
12
|
-
java_import "com.maxmind.geoip2.record.City"
|
13
|
-
java_import "com.maxmind.geoip2.record.Postal"
|
14
|
-
java_import "com.maxmind.geoip2.record.Location"
|
15
|
-
java_import "com.maxmind.db.CHMCache"
|
16
|
-
|
17
|
-
def suppress_all_warnings
|
18
|
-
old_verbose = $VERBOSE
|
19
|
-
begin
|
20
|
-
$VERBOSE = nil
|
21
|
-
yield if block_given?
|
22
|
-
ensure
|
23
|
-
# always re-set to old value, even if block raises an exception
|
24
|
-
$VERBOSE = old_verbose
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
# create a new instance of the Java class File without shadowing the Ruby version of the File class
|
29
|
-
module JavaIO
|
30
|
-
include_package "java.io"
|
31
|
-
end
|
32
|
-
|
33
|
-
|
34
7
|
# The GeoIP filter adds information about the geographical location of IP addresses,
|
35
8
|
# based on data from the Maxmind GeoLite2 database.
|
36
9
|
#
|
@@ -46,9 +19,16 @@ end
|
|
46
19
|
# and the flexibility of having GeoJSON for all other applications (like Kibana's
|
47
20
|
# map visualization).
|
48
21
|
#
|
49
|
-
#
|
22
|
+
# [NOTE]
|
23
|
+
# --
|
24
|
+
# This product includes GeoLite2 data created by MaxMind, available from
|
50
25
|
# http://www.maxmind.com. This database is licensed under
|
51
|
-
# http://creativecommons.org/licenses/by-sa/4.0/[Creative Commons Attribution-ShareAlike 4.0 International License]
|
26
|
+
# http://creativecommons.org/licenses/by-sa/4.0/[Creative Commons Attribution-ShareAlike 4.0 International License].
|
27
|
+
#
|
28
|
+
# Versions 4.0.0 and later of the GeoIP filter use the MaxMind GeoLite2 database
|
29
|
+
# and support both IPv4 and IPv6 lookups. Versions prior to 4.0.0 use the legacy
|
30
|
+
# MaxMind GeoLite database and support IPv4 lookups only.
|
31
|
+
# --
|
52
32
|
|
53
33
|
class LogStash::Filters::GeoIP < LogStash::Filters::Base
|
54
34
|
config_name "geoip"
|
@@ -71,11 +51,7 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
|
|
71
51
|
# For the built-in GeoLite2 City database, the following are available:
|
72
52
|
# `city_name`, `continent_code`, `country_code2`, `country_code3`, `country_name`,
|
73
53
|
# `dma_code`, `ip`, `latitude`, `longitude`, `postal_code`, `region_name` and `timezone`.
|
74
|
-
config :fields, :validate => :array
|
75
|
-
'country_code2', 'country_code3', 'country_name',
|
76
|
-
'dma_code', 'ip', 'latitude',
|
77
|
-
'longitude', 'postal_code', 'region_name',
|
78
|
-
'region_code', 'timezone', 'location']
|
54
|
+
config :fields, :validate => :array
|
79
55
|
|
80
56
|
# Specify the field into which Logstash should store the geoip data.
|
81
57
|
# This can be useful, for example, if you have `src_ip` and `dst_ip` fields and
|
@@ -120,111 +96,34 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
|
|
120
96
|
# of the same geoip_type share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit
|
121
97
|
# to having multiple caches for different instances at different points in the pipeline, that would just increase the
|
122
98
|
# number of cache misses and waste memory.
|
123
|
-
config :lru_cache_size, :validate => :number, :default => 1000
|
99
|
+
config :lru_cache_size, :validate => :number, :default => 1000, :deprecated => "This field has been deprecated in favor of cache_size."
|
124
100
|
|
125
101
|
# Tags the event on failure to look up geo information. This can be used in later analysis.
|
126
102
|
config :tag_on_failure, :validate => :array, :default => ["_geoip_lookup_failure"]
|
127
103
|
|
128
104
|
public
|
129
105
|
def register
|
130
|
-
|
131
|
-
|
132
|
-
@database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"GeoLite2-City.mmdb")).first
|
106
|
+
if @database.nil?
|
107
|
+
@database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"GeoLite2-City.mmdb")).first
|
133
108
|
|
134
|
-
|
135
|
-
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
139
|
-
@logger.info("Using geoip database", :path => @database)
|
140
|
-
|
141
|
-
db_file = JavaIO::File.new(@database)
|
142
|
-
begin
|
143
|
-
@parser = DatabaseReader::Builder.new(db_file).withCache(CHMCache.new(@cache_size)).build();
|
144
|
-
rescue Java::ComMaxmindDb::InvalidDatabaseException => e
|
145
|
-
@logger.error("The GeoLite2 MMDB database provided is invalid or corrupted.", :exception => e, :field => @source)
|
146
|
-
raise e
|
109
|
+
if @database.nil? || !File.exists?(@database)
|
110
|
+
raise "You must specify 'database => ...' in your geoip filter (I looked for '#{@database}')"
|
147
111
|
end
|
148
112
|
end
|
113
|
+
|
114
|
+
@logger.info("Using geoip database", :path => @database)
|
115
|
+
|
116
|
+
@geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, @fields, @database, @cache_size)
|
149
117
|
end # def register
|
150
118
|
|
151
119
|
public
|
152
120
|
def filter(event)
|
153
121
|
return unless filter?(event)
|
154
|
-
|
155
|
-
begin
|
156
|
-
ip = event.get(@source)
|
157
|
-
ip = ip.first if ip.is_a? Array
|
158
|
-
geo_data_hash = Hash.new
|
159
|
-
ip_address = InetAddress.getByName(ip)
|
160
|
-
response = @parser.city(ip_address)
|
161
|
-
populate_geo_data(response, ip_address, geo_data_hash)
|
162
|
-
rescue com.maxmind.geoip2.exception.AddressNotFoundException => e
|
163
|
-
@logger.debug("IP not found!", :exception => e, :field => @source, :event => event)
|
164
|
-
rescue java.net.UnknownHostException => e
|
165
|
-
@logger.error("IP Field contained invalid IP address or hostname", :exception => e, :field => @source, :event => event)
|
166
|
-
rescue Exception => e
|
167
|
-
@logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @source, :event => event)
|
168
|
-
# Dont' swallow this, bubble up for unknown issue
|
169
|
-
raise e
|
170
|
-
end
|
171
|
-
|
172
|
-
if apply_geodata(geo_data_hash, event)
|
122
|
+
if @geoipfilter.handleEvent(event)
|
173
123
|
filter_matched(event)
|
174
124
|
else
|
175
125
|
tag_unsuccessful_lookup(event)
|
176
126
|
end
|
177
|
-
end # def filter
|
178
|
-
|
179
|
-
def populate_geo_data(response, ip_address, geo_data_hash)
|
180
|
-
country = response.getCountry()
|
181
|
-
subdivision = response.getMostSpecificSubdivision()
|
182
|
-
city = response.getCity()
|
183
|
-
postal = response.getPostal()
|
184
|
-
location = response.getLocation()
|
185
|
-
|
186
|
-
# if location is empty, there is no point populating geo data
|
187
|
-
# and most likely all other fields are empty as well
|
188
|
-
if location.getLatitude().nil? && location.getLongitude().nil?
|
189
|
-
return
|
190
|
-
end
|
191
|
-
|
192
|
-
@fields.each do |field|
|
193
|
-
case field
|
194
|
-
when "city_name"
|
195
|
-
geo_data_hash["city_name"] = city.getName()
|
196
|
-
when "country_name"
|
197
|
-
geo_data_hash["country_name"] = country.getName()
|
198
|
-
when "continent_code"
|
199
|
-
geo_data_hash["continent_code"] = response.getContinent().getCode()
|
200
|
-
when "continent_name"
|
201
|
-
geo_data_hash["continent_name"] = response.getContinent().getName()
|
202
|
-
when "country_code2"
|
203
|
-
geo_data_hash["country_code2"] = country.getIsoCode()
|
204
|
-
when "country_code3"
|
205
|
-
geo_data_hash["country_code3"] = country.getIsoCode()
|
206
|
-
when "ip"
|
207
|
-
geo_data_hash["ip"] = ip_address.getHostAddress()
|
208
|
-
when "postal_code"
|
209
|
-
geo_data_hash["postal_code"] = postal.getCode()
|
210
|
-
when "dma_code"
|
211
|
-
geo_data_hash["dma_code"] = location.getMetroCode()
|
212
|
-
when "region_name"
|
213
|
-
geo_data_hash["region_name"] = subdivision.getName()
|
214
|
-
when "region_code"
|
215
|
-
geo_data_hash["region_code"] = subdivision.getIsoCode()
|
216
|
-
when "timezone"
|
217
|
-
geo_data_hash["timezone"] = location.getTimeZone()
|
218
|
-
when "location"
|
219
|
-
geo_data_hash["location"] = [ location.getLongitude(), location.getLatitude() ]
|
220
|
-
when "latitude"
|
221
|
-
geo_data_hash["latitude"] = location.getLatitude()
|
222
|
-
when "longitude"
|
223
|
-
geo_data_hash["longitude"] = location.getLongitude()
|
224
|
-
else
|
225
|
-
raise Exception.new("[#{field}] is not a supported field option.")
|
226
|
-
end
|
227
|
-
end
|
228
127
|
end
|
229
128
|
|
230
129
|
def tag_unsuccessful_lookup(event)
|
@@ -232,20 +131,4 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
|
|
232
131
|
@tag_on_failure.each{|tag| event.tag(tag)}
|
233
132
|
end
|
234
133
|
|
235
|
-
def apply_geodata(geo_data_hash, event)
|
236
|
-
# don't do anything more if the lookup result is nil?
|
237
|
-
return false if geo_data_hash.nil?
|
238
|
-
# only do event.set(@target) if the lookup result is not nil
|
239
|
-
event.set(@target, {}) if event.get(@target).nil?
|
240
|
-
# don't do anything more if the lookup result is empty?
|
241
|
-
return false if geo_data_hash.empty?
|
242
|
-
geo_data_hash.each do |key, value|
|
243
|
-
if @fields.include?(key) && value
|
244
|
-
# can't dup numerics
|
245
|
-
event.set("[#{@target}][#{key}]", value.is_a?(Numeric) ? value : value.dup)
|
246
|
-
end
|
247
|
-
end # geo_data_hash.each
|
248
|
-
true
|
249
|
-
end
|
250
|
-
|
251
134
|
end # class LogStash::Filters::GeoIP
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
def suppress_all_warnings
|
3
|
+
old_verbose = $VERBOSE
|
4
|
+
begin
|
5
|
+
$VERBOSE = nil
|
6
|
+
yield if block_given?
|
7
|
+
ensure
|
8
|
+
# always re-set to old value, even if block raises an exception
|
9
|
+
$VERBOSE = old_verbose
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
# create a new instance of the Java class File without shadowing the Ruby version of the File class
|
14
|
+
module JavaIO
|
15
|
+
include_package "java.io"
|
16
|
+
end
|
17
|
+
|
@@ -1,18 +1,18 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-geoip'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.1.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "$summary"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ["Elastic"]
|
9
9
|
s.email = 'info@elastic.co'
|
10
10
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
11
|
-
s.require_paths = ["lib"]
|
12
11
|
s.platform = "java"
|
12
|
+
s.require_paths = ["lib", "vendor/jar-dependencies"]
|
13
13
|
|
14
14
|
# Files
|
15
|
-
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT', 'maxmind-db-NOTICE.txt']
|
15
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*', 'vendor/jar-dependencies/**/*.jar', '*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT', 'maxmind-db-NOTICE.txt', 'docs/**/*']
|
16
16
|
|
17
17
|
# Tests
|
18
18
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
@@ -22,12 +22,6 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
# Gem dependencies
|
24
24
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
25
|
-
|
26
|
-
s.requirements << "jar com.maxmind.geoip2:geoip2, 2.5.0, :exclusions=> [com.google.http-client:google-http-client]"
|
27
|
-
|
28
|
-
s.add_development_dependency "jar-dependencies"
|
29
|
-
|
30
|
-
s.add_development_dependency 'ruby-maven', '~> 3.3'
|
31
|
-
|
32
25
|
s.add_development_dependency 'logstash-devutils'
|
26
|
+
s.add_development_dependency 'benchmark-ips'
|
33
27
|
end
|
data/spec/filters/geoip_spec.rb
CHANGED
@@ -97,7 +97,7 @@ describe LogStash::Filters::GeoIP do
|
|
97
97
|
|
98
98
|
it "should set other subfields of 'target' properly" do
|
99
99
|
expect(event.get("target").to_hash.keys.sort).to eq(["city_name", "ip", "region_name"])
|
100
|
-
expect(event.get("[target][city_name]")).to eq("
|
100
|
+
expect(event.get("[target][city_name]")).to eq("Worcester")
|
101
101
|
expect(event.get("[target][region_name]")).to eq("Massachusetts")
|
102
102
|
end
|
103
103
|
|
@@ -251,13 +251,6 @@ describe LogStash::Filters::GeoIP do
|
|
251
251
|
|
252
252
|
end
|
253
253
|
|
254
|
-
context "should return the correct source field in the logging message" do
|
255
|
-
sample("ip" => "-") do
|
256
|
-
expect(LogStash::Filters::GeoIP.logger).to receive(:error).with(anything, include(:field => "ip"))
|
257
|
-
subject
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
254
|
end
|
262
255
|
|
263
256
|
describe "an invalid database" do
|
@@ -272,8 +265,7 @@ describe LogStash::Filters::GeoIP do
|
|
272
265
|
|
273
266
|
context "should return the correct sourcefield in the logging message" do
|
274
267
|
sample("ip" => "8.8.8.8") do
|
275
|
-
expect
|
276
|
-
expect { subject }.to raise_error
|
268
|
+
expect { subject }.to raise_error(java.lang.IllegalArgumentException, "The database provided is invalid or corrupted.")
|
277
269
|
end
|
278
270
|
end
|
279
271
|
end
|
data/vendor/GeoLite2-City.mmdb
CHANGED
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-geoip
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.1.1
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-05-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -36,7 +36,7 @@ dependencies:
|
|
36
36
|
- - ">="
|
37
37
|
- !ruby/object:Gem::Version
|
38
38
|
version: '0'
|
39
|
-
name:
|
39
|
+
name: logstash-devutils
|
40
40
|
prerelease: false
|
41
41
|
type: :development
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -44,27 +44,13 @@ dependencies:
|
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
requirement: !ruby/object:Gem::Requirement
|
49
|
-
requirements:
|
50
|
-
- - "~>"
|
51
|
-
- !ruby/object:Gem::Version
|
52
|
-
version: '3.3'
|
53
|
-
name: ruby-maven
|
54
|
-
prerelease: false
|
55
|
-
type: :development
|
56
|
-
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
requirements:
|
58
|
-
- - "~>"
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: '3.3'
|
61
47
|
- !ruby/object:Gem::Dependency
|
62
48
|
requirement: !ruby/object:Gem::Requirement
|
63
49
|
requirements:
|
64
50
|
- - ">="
|
65
51
|
- !ruby/object:Gem::Version
|
66
52
|
version: '0'
|
67
|
-
name:
|
53
|
+
name: benchmark-ips
|
68
54
|
prerelease: false
|
69
55
|
type: :development
|
70
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -84,22 +70,17 @@ files:
|
|
84
70
|
- LICENSE
|
85
71
|
- NOTICE.TXT
|
86
72
|
- README.md
|
87
|
-
-
|
88
|
-
- lib/com/fasterxml/jackson/core/jackson-core/2.6.4/jackson-core-2.6.4.jar
|
89
|
-
- lib/com/fasterxml/jackson/core/jackson-databind/2.6.4/jackson-databind-2.6.4.jar
|
90
|
-
- lib/com/maxmind/db/maxmind-db/1.1.0/maxmind-db-1.1.0.jar
|
91
|
-
- lib/com/maxmind/geoip2/geoip2/2.5.0/geoip2-2.5.0.jar
|
73
|
+
- docs/index.asciidoc
|
92
74
|
- lib/logstash-filter-geoip_jars.rb
|
93
75
|
- lib/logstash/filters/geoip.rb
|
76
|
+
- lib/logstash/filters/geoip/patch.rb
|
94
77
|
- logstash-filter-geoip.gemspec
|
95
78
|
- maxmind-db-NOTICE.txt
|
96
79
|
- spec/filters/geoip_spec.rb
|
97
80
|
- vendor/GeoLite2-City.mmdb
|
98
|
-
- vendor/jar-dependencies/
|
99
|
-
- vendor/jar-dependencies/
|
100
|
-
- vendor/jar-dependencies/
|
101
|
-
- vendor/jar-dependencies/runtime-jars/jackson-databind-2.6.4.jar
|
102
|
-
- vendor/jar-dependencies/runtime-jars/maxmind-db-1.1.0.jar
|
81
|
+
- vendor/jar-dependencies/com/maxmind/db/maxmind-db/1.2.1/maxmind-db-1.2.1.jar
|
82
|
+
- vendor/jar-dependencies/com/maxmind/geoip2/geoip2/2.8.0/geoip2-2.8.0.jar
|
83
|
+
- vendor/jar-dependencies/org/logstash/filters/logstash-filter-geoip/4.1.0/logstash-filter-geoip-4.1.0.jar
|
103
84
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
104
85
|
licenses:
|
105
86
|
- Apache License (2.0)
|
@@ -110,6 +91,7 @@ post_install_message:
|
|
110
91
|
rdoc_options: []
|
111
92
|
require_paths:
|
112
93
|
- lib
|
94
|
+
- vendor/jar-dependencies
|
113
95
|
required_ruby_version: !ruby/object:Gem::Requirement
|
114
96
|
requirements:
|
115
97
|
- - ">="
|
@@ -120,8 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
120
102
|
- - ">="
|
121
103
|
- !ruby/object:Gem::Version
|
122
104
|
version: '0'
|
123
|
-
requirements:
|
124
|
-
- jar com.maxmind.geoip2:geoip2, 2.5.0, :exclusions=> [com.google.http-client:google-http-client]
|
105
|
+
requirements: []
|
125
106
|
rubyforge_project:
|
126
107
|
rubygems_version: 2.4.8
|
127
108
|
signing_key:
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|