logstash-filter-ip2location 2.1.0 → 2.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +11 -11
- data/LICENSE +12 -12
- data/README.md +121 -121
- data/lib/logstash-filter-ip2location_jars.rb +3 -3
- data/lib/logstash/filters/ip2location.rb +161 -161
- data/logstash-filter-ip2location.gemspec +25 -25
- data/spec/filters/ip2location_spec.rb +31 -31
- data/spec/spec_helper.rb +2 -2
- data/vendor/IP2LOCATION-LITE-DB1.IPV6.BIN +0 -0
- data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0.jar +0 -0
- data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/META-INF/MANIFEST.MF +3 -0
- data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/com/ip2location/IP2Location.class +0 -0
- data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/com/ip2location/IPResult.class +0 -0
- data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/com/ip2location/MetaData.class +0 -0
- data/vendor/jar-dependencies/org/logstash/filters/logstash-filter-ip2location/{2.1.0/logstash-filter-ip2location-2.1.0.jar → 2.1.1/logstash-filter-ip2location-2.1.1.jar} +0 -0
- metadata +11 -6
- data/vendor/IP2LOCATION-LITE-DB3.IPV6.BIN +0 -0
- data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.1.0/ip2location-8.1.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3243f5ebb4f5321ec00c9342557c45437ba22e4527d63e93307412b49b5bde50
|
4
|
+
data.tar.gz: 21198a5b4664a8b88924020d71a547b098107a77c72d80e62002c9539c1f8a7a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2ed700bc40d944621685a12b8f92db4aa08c669c498c384ef67e504d12a7e5dccd2b3df33950085e95e9dc6abffe4e7d69230d47c99ab696c5c11760793d89a0
|
7
|
+
data.tar.gz: e3f27377f860a8009bf1c9663b2ff05e4b1a3a5ce074933783867a950412a73dffaf97f63a8eb28fb36863943a8a2c1b2b86d8c8e22e4d3651d23ae76f24aca4
|
data/Gemfile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
source 'https://rubygems.org'
|
2
|
-
|
3
|
-
gemspec
|
4
|
-
|
5
|
-
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
-
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
-
|
8
|
-
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
-
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
-
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
-
end
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/LICENSE
CHANGED
@@ -1,13 +1,13 @@
|
|
1
|
-
Copyright (c)
|
2
|
-
|
3
|
-
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
you may not use this file except in compliance with the License.
|
5
|
-
You may obtain a copy of the License at
|
6
|
-
|
7
|
-
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
|
9
|
-
Unless required by applicable law or agreed to in writing, software
|
10
|
-
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
See the License for the specific language governing permissions and
|
1
|
+
Copyright (c) 2019 IP2Location.com
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
13
|
limitations under the License.
|
data/README.md
CHANGED
@@ -1,121 +1,121 @@
|
|
1
|
-
# IP2Location Filter Plugin
|
2
|
-
This is IP2Location filter plugin for Logstash that enables Logstash's users to add geolocation information such as country, region, city, latitude, longitude, ZIP code, time zone, Internet Service Provider (ISP), domain name, connection speed, IDD code, area code, weather station code, weather station name, mobile country code (MCC), mobile network code (MNC), mobile brand, elevation, and usage type by IP address. The library reads the geo location information from **IP2Location BIN data** file.
|
3
|
-
|
4
|
-
Supported IPv4 and IPv6 address.
|
5
|
-
|
6
|
-
For the methods to use IP2Location filter plugin with Elastic Stack (Elasticsearch, Filebeat, Logstash, and Kibana), please take a look on this [tutorial](https://www.ip2location.com/tutorials/how-to-use-ip2location-filter-plugin-with-elastic-stack).
|
7
|
-
|
8
|
-
|
9
|
-
## Dependencies (IP2LOCATION BIN DATA FILE)
|
10
|
-
This plugin requires IP2Location BIN data file to function. You may download the BIN data file at
|
11
|
-
* IP2Location LITE BIN Data (Free): https://lite.ip2location.com
|
12
|
-
* IP2Location Commercial BIN Data (Commercial): https://www.ip2location.com
|
13
|
-
|
14
|
-
|
15
|
-
## Installation
|
16
|
-
Install this plugin by the following code:
|
17
|
-
```
|
18
|
-
bin/logstash-plugin install logstash-filter-ip2location
|
19
|
-
```
|
20
|
-
|
21
|
-
|
22
|
-
## Config File Example 1
|
23
|
-
```
|
24
|
-
input {
|
25
|
-
beats {
|
26
|
-
port => "5043"
|
27
|
-
}
|
28
|
-
}
|
29
|
-
|
30
|
-
filter {
|
31
|
-
grok {
|
32
|
-
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
33
|
-
}
|
34
|
-
ip2location {
|
35
|
-
source => "clientip"
|
36
|
-
}
|
37
|
-
}
|
38
|
-
|
39
|
-
|
40
|
-
output {
|
41
|
-
elasticsearch {
|
42
|
-
hosts => [ "localhost:9200" ]
|
43
|
-
}
|
44
|
-
}
|
45
|
-
```
|
46
|
-
|
47
|
-
|
48
|
-
## Config File Example 2
|
49
|
-
```
|
50
|
-
input {
|
51
|
-
beats {
|
52
|
-
port => "5043"
|
53
|
-
}
|
54
|
-
}
|
55
|
-
|
56
|
-
filter {
|
57
|
-
grok {
|
58
|
-
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
59
|
-
}
|
60
|
-
ip2location {
|
61
|
-
source => "clientip"
|
62
|
-
# Set path to the database located
|
63
|
-
database => "IP2LOCATION_BIN_DATABASE_FILESYSTEM_PATH"
|
64
|
-
# Enable memory mapped to be used
|
65
|
-
use_memory_mapped => true
|
66
|
-
}
|
67
|
-
}
|
68
|
-
|
69
|
-
|
70
|
-
output {
|
71
|
-
elasticsearch {
|
72
|
-
hosts => [ "localhost:9200" ]
|
73
|
-
}
|
74
|
-
}
|
75
|
-
```
|
76
|
-
|
77
|
-
|
78
|
-
## IP2Location Filter Configuration
|
79
|
-
|Setting|Input type|Required|
|
80
|
-
|---|---|---|
|
81
|
-
|source|string|Yes|
|
82
|
-
|database|a valid filesystem path|No|
|
83
|
-
|use_memory_mapped|boolean|No|
|
84
|
-
|
85
|
-
* **source** field is a required setting that containing the IP address or hostname to get the ip information.
|
86
|
-
* **database** field is an optional setting that containing the path to the IP2Location BIN database file.
|
87
|
-
* **use_memory_mapped** field is an optional setting that used to allow user to enable the use of memory mapped file. Default value is false.
|
88
|
-
|
89
|
-
|
90
|
-
## Sample Output
|
91
|
-
|Field|Description|
|
92
|
-
|---|---|
|
93
|
-
|ip2location.area_code|the varying length number assigned to geographic areas for call between cities|
|
94
|
-
|ip2location.city|the city name|
|
95
|
-
|ip2location.country_long|the country name based on ISO 3166|
|
96
|
-
|ip2location.country_short|the two-character country code based on ISO 3166|
|
97
|
-
|ip2location.domain|the Internet domain name associated to IP address range|
|
98
|
-
|ip2location.elevation|the elevation|
|
99
|
-
|ip2location.idd_code|the IDD prefix to call the city from another country|
|
100
|
-
|ip2location.ip_address|the IP address|
|
101
|
-
|ip2location.isp|the Internet Service Provider (ISP) name|
|
102
|
-
|ip2location.latitude|the city latitude|
|
103
|
-
|ip2location.longitude|the city longitude|
|
104
|
-
|ip2location.mcc|the mobile country code|
|
105
|
-
|ip2location.mnc|mobile network code|
|
106
|
-
|ip2location.mobile_brand|the mobile brand|
|
107
|
-
|ip2location.net_speed|the Internet Connection Speed (DIAL) DIAL-UP,(DSL) DSL/CABLE or(COMP) COMPANY|
|
108
|
-
|ip2location.region|the region or state name|
|
109
|
-
|ip2location.time_zone|the Time zone in UTC (Coordinated Universal Time)|
|
110
|
-
|ip2location.usage_type|the usage type|
|
111
|
-
|ip2location.weather_station_code|the special code to identify the nearest weather observation station|
|
112
|
-
|ip2location.weather_station_name|the name of the nearest weather observation station|
|
113
|
-
|ip2location.zip_code|the ZIP code|
|
114
|
-
|
115
|
-
![Example of data](https://www.ip2location.com/assets/img/logstash-filter-ip2location-screenshot2.png)
|
116
|
-
|
117
|
-
|
118
|
-
## Support
|
119
|
-
Email: support@ip2location.com
|
120
|
-
|
121
|
-
URL: [https://www.ip2location.com](https://www.ip2location.com)
|
1
|
+
# IP2Location Filter Plugin
|
2
|
+
This is IP2Location filter plugin for Logstash that enables Logstash's users to add geolocation information such as country, region, city, latitude, longitude, ZIP code, time zone, Internet Service Provider (ISP), domain name, connection speed, IDD code, area code, weather station code, weather station name, mobile country code (MCC), mobile network code (MNC), mobile brand, elevation, and usage type by IP address. The library reads the geo location information from **IP2Location BIN data** file.
|
3
|
+
|
4
|
+
Supported IPv4 and IPv6 address.
|
5
|
+
|
6
|
+
For the methods to use IP2Location filter plugin with Elastic Stack (Elasticsearch, Filebeat, Logstash, and Kibana), please take a look on this [tutorial](https://www.ip2location.com/tutorials/how-to-use-ip2location-filter-plugin-with-elastic-stack).
|
7
|
+
|
8
|
+
|
9
|
+
## Dependencies (IP2LOCATION BIN DATA FILE)
|
10
|
+
This plugin requires IP2Location BIN data file to function. You may download the BIN data file at
|
11
|
+
* IP2Location LITE BIN Data (Free): https://lite.ip2location.com
|
12
|
+
* IP2Location Commercial BIN Data (Commercial): https://www.ip2location.com
|
13
|
+
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
Install this plugin by the following code:
|
17
|
+
```
|
18
|
+
bin/logstash-plugin install logstash-filter-ip2location
|
19
|
+
```
|
20
|
+
|
21
|
+
|
22
|
+
## Config File Example 1
|
23
|
+
```
|
24
|
+
input {
|
25
|
+
beats {
|
26
|
+
port => "5043"
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
filter {
|
31
|
+
grok {
|
32
|
+
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
33
|
+
}
|
34
|
+
ip2location {
|
35
|
+
source => "clientip"
|
36
|
+
}
|
37
|
+
}
|
38
|
+
|
39
|
+
|
40
|
+
output {
|
41
|
+
elasticsearch {
|
42
|
+
hosts => [ "localhost:9200" ]
|
43
|
+
}
|
44
|
+
}
|
45
|
+
```
|
46
|
+
|
47
|
+
|
48
|
+
## Config File Example 2
|
49
|
+
```
|
50
|
+
input {
|
51
|
+
beats {
|
52
|
+
port => "5043"
|
53
|
+
}
|
54
|
+
}
|
55
|
+
|
56
|
+
filter {
|
57
|
+
grok {
|
58
|
+
match => { "message" => "%{COMBINEDAPACHELOG}"}
|
59
|
+
}
|
60
|
+
ip2location {
|
61
|
+
source => "clientip"
|
62
|
+
# Set path to the database located
|
63
|
+
database => "IP2LOCATION_BIN_DATABASE_FILESYSTEM_PATH"
|
64
|
+
# Enable memory mapped to be used
|
65
|
+
use_memory_mapped => true
|
66
|
+
}
|
67
|
+
}
|
68
|
+
|
69
|
+
|
70
|
+
output {
|
71
|
+
elasticsearch {
|
72
|
+
hosts => [ "localhost:9200" ]
|
73
|
+
}
|
74
|
+
}
|
75
|
+
```
|
76
|
+
|
77
|
+
|
78
|
+
## IP2Location Filter Configuration
|
79
|
+
|Setting|Input type|Required|
|
80
|
+
|---|---|---|
|
81
|
+
|source|string|Yes|
|
82
|
+
|database|a valid filesystem path|No|
|
83
|
+
|use_memory_mapped|boolean|No|
|
84
|
+
|
85
|
+
* **source** field is a required setting that containing the IP address or hostname to get the ip information.
|
86
|
+
* **database** field is an optional setting that containing the path to the IP2Location BIN database file.
|
87
|
+
* **use_memory_mapped** field is an optional setting that used to allow user to enable the use of memory mapped file. Default value is false.
|
88
|
+
|
89
|
+
|
90
|
+
## Sample Output
|
91
|
+
|Field|Description|
|
92
|
+
|---|---|
|
93
|
+
|ip2location.area_code|the varying length number assigned to geographic areas for call between cities|
|
94
|
+
|ip2location.city|the city name|
|
95
|
+
|ip2location.country_long|the country name based on ISO 3166|
|
96
|
+
|ip2location.country_short|the two-character country code based on ISO 3166|
|
97
|
+
|ip2location.domain|the Internet domain name associated to IP address range|
|
98
|
+
|ip2location.elevation|the elevation|
|
99
|
+
|ip2location.idd_code|the IDD prefix to call the city from another country|
|
100
|
+
|ip2location.ip_address|the IP address|
|
101
|
+
|ip2location.isp|the Internet Service Provider (ISP) name|
|
102
|
+
|ip2location.latitude|the city latitude|
|
103
|
+
|ip2location.longitude|the city longitude|
|
104
|
+
|ip2location.mcc|the mobile country code|
|
105
|
+
|ip2location.mnc|mobile network code|
|
106
|
+
|ip2location.mobile_brand|the mobile brand|
|
107
|
+
|ip2location.net_speed|the Internet Connection Speed (DIAL) DIAL-UP,(DSL) DSL/CABLE or(COMP) COMPANY|
|
108
|
+
|ip2location.region|the region or state name|
|
109
|
+
|ip2location.time_zone|the Time zone in UTC (Coordinated Universal Time)|
|
110
|
+
|ip2location.usage_type|the usage type|
|
111
|
+
|ip2location.weather_station_code|the special code to identify the nearest weather observation station|
|
112
|
+
|ip2location.weather_station_name|the name of the nearest weather observation station|
|
113
|
+
|ip2location.zip_code|the ZIP code|
|
114
|
+
|
115
|
+
![Example of data](https://www.ip2location.com/assets/img/logstash-filter-ip2location-screenshot2.png)
|
116
|
+
|
117
|
+
|
118
|
+
## Support
|
119
|
+
Email: support@ip2location.com
|
120
|
+
|
121
|
+
URL: [https://www.ip2location.com](https://www.ip2location.com)
|
@@ -1,3 +1,3 @@
|
|
1
|
-
require 'jar_dependencies'
|
2
|
-
require_jar('com.ip2location.ip2location', 'ip2location', '8.
|
3
|
-
require_jar('org.logstash.filters', 'logstash-filter-ip2location', '2.1.
|
1
|
+
require 'jar_dependencies'
|
2
|
+
require_jar('com.ip2location.ip2location', 'ip2location', '8.2.0')
|
3
|
+
require_jar('org.logstash.filters', 'logstash-filter-ip2location', '2.1.1')
|
@@ -1,161 +1,161 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/filters/base"
|
3
|
-
require "logstash/namespace"
|
4
|
-
|
5
|
-
require "logstash-filter-ip2location_jars"
|
6
|
-
|
7
|
-
require 'thread'
|
8
|
-
|
9
|
-
|
10
|
-
class LogStash::Filters::IP2Location < LogStash::Filters::Base
|
11
|
-
config_name "ip2location"
|
12
|
-
|
13
|
-
# The path to the IP2Location.BIN database file which Logstash should use.
|
14
|
-
# If not specified, this will default to the IP2LOCATION-LITE-
|
15
|
-
config :database, :validate => :path
|
16
|
-
|
17
|
-
# The field containing the IP address.
|
18
|
-
# If this field is an array, only the first value will be used.
|
19
|
-
config :source, :validate => :string, :required => true
|
20
|
-
|
21
|
-
# The field used to define iplocation as target.
|
22
|
-
config :target, :validate => :string, :default => 'ip2location'
|
23
|
-
|
24
|
-
# The field used to allow user to enable the use of memory mapped file.
|
25
|
-
config :use_memory_mapped, :validate => :boolean, :default => false
|
26
|
-
|
27
|
-
# The field used to define the size of the cache. It is not required and the default value is 10 000
|
28
|
-
config :cache_size, :validate => :number, :required => false, :default => 10_000
|
29
|
-
|
30
|
-
public
|
31
|
-
def register
|
32
|
-
if @database.nil?
|
33
|
-
@database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"IP2LOCATION-LITE-
|
34
|
-
|
35
|
-
if @database.nil? || !File.exists?(@database)
|
36
|
-
raise "You must specify 'database => ...' in your ip2location filter (I looked for '#{@database}')"
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
@logger.info("Using ip2location database", :path => @database)
|
41
|
-
|
42
|
-
@ip2locationfilter = org.logstash.filters.IP2LocationFilter.new(@source, @target, @database, @use_memory_mapped)
|
43
|
-
end
|
44
|
-
|
45
|
-
public
|
46
|
-
def filter(event)
|
47
|
-
ip = event.get(@source)
|
48
|
-
|
49
|
-
return unless filter?(event)
|
50
|
-
if value = Cache.find(event, ip, @ip2locationfilter, @cache_size).get('ip2location')
|
51
|
-
event.set('ip2location', value)
|
52
|
-
filter_matched(event)
|
53
|
-
else
|
54
|
-
tag_iplookup_unsuccessful(event)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def tag_iplookup_unsuccessful(event)
|
59
|
-
@logger.debug? && @logger.debug("IP #{event.get(@source)} was not found in the database", :event => event)
|
60
|
-
end
|
61
|
-
|
62
|
-
end # class LogStash::Filters::IP2Location
|
63
|
-
|
64
|
-
class OrderedHash
|
65
|
-
ONE = 1
|
66
|
-
|
67
|
-
attr_reader :times_queried # ip -> times queried
|
68
|
-
attr_reader :hash
|
69
|
-
|
70
|
-
def initialize
|
71
|
-
@times_queried = Hash.new(0) # ip -> times queried
|
72
|
-
@hash = {} # number of hits -> array of ips
|
73
|
-
end
|
74
|
-
|
75
|
-
def add(key)
|
76
|
-
hash[ONE] ||= []
|
77
|
-
hash[ONE] << key
|
78
|
-
times_queried[key] = ONE
|
79
|
-
end
|
80
|
-
|
81
|
-
def reorder(key)
|
82
|
-
number_of_queries = times_queried[key]
|
83
|
-
|
84
|
-
hash[number_of_queries].delete(key)
|
85
|
-
hash.delete(number_of_queries) if hash[number_of_queries].empty?
|
86
|
-
|
87
|
-
hash[number_of_queries + 1] ||= []
|
88
|
-
hash[number_of_queries + 1] << key
|
89
|
-
end
|
90
|
-
|
91
|
-
def increment(key)
|
92
|
-
add(key) unless times_queried.has_key?(key)
|
93
|
-
reorder(key)
|
94
|
-
times_queried[key] += 1
|
95
|
-
end
|
96
|
-
|
97
|
-
def delete_least_used
|
98
|
-
first_pile_with_someting.shift.tap { |key| times_queried.delete(key) }
|
99
|
-
end
|
100
|
-
|
101
|
-
def first_pile_with_someting
|
102
|
-
hash[hash.keys.min]
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
class Cache
|
107
|
-
ONE_DAY_IN_SECONDS = 86_400
|
108
|
-
|
109
|
-
@cache = {} # ip -> event
|
110
|
-
@timestamps = {} # ip -> time of caching
|
111
|
-
@times_queried = OrderedHash.new # ip -> times queried
|
112
|
-
@mutex = Mutex.new
|
113
|
-
|
114
|
-
class << self
|
115
|
-
attr_reader :cache
|
116
|
-
attr_reader :timestamps
|
117
|
-
attr_reader :times_queried
|
118
|
-
|
119
|
-
|
120
|
-
def find(event, ip, filter, cache_size)
|
121
|
-
synchronize do
|
122
|
-
if cache.has_key?(ip)
|
123
|
-
refresh_event(ip) if too_old?(ip)
|
124
|
-
else
|
125
|
-
if cache_full?(cache_size)
|
126
|
-
make_room
|
127
|
-
end
|
128
|
-
cache_event(event, ip, filter)
|
129
|
-
end
|
130
|
-
times_queried.increment(ip)
|
131
|
-
cache[ip]
|
132
|
-
end
|
133
|
-
end
|
134
|
-
|
135
|
-
def too_old?(ip)
|
136
|
-
timestamps[ip] < Time.now - ONE_DAY_IN_SECONDS
|
137
|
-
end
|
138
|
-
|
139
|
-
def make_room
|
140
|
-
key = times_queried.delete_least_used
|
141
|
-
cache.delete(key)
|
142
|
-
timestamps.delete(key)
|
143
|
-
end
|
144
|
-
|
145
|
-
def cache_full?(cache_size)
|
146
|
-
cache.size >= cache_size
|
147
|
-
end
|
148
|
-
|
149
|
-
def cache_event(event, ip, filter)
|
150
|
-
filter.handleEvent(event)
|
151
|
-
cache[ip] = event
|
152
|
-
timestamps[ip] = Time.now
|
153
|
-
end
|
154
|
-
|
155
|
-
def synchronize(&block)
|
156
|
-
@mutex.synchronize(&block)
|
157
|
-
end
|
158
|
-
|
159
|
-
alias_method :refresh_event, :cache_event
|
160
|
-
end
|
161
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/filters/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
|
5
|
+
require "logstash-filter-ip2location_jars"
|
6
|
+
|
7
|
+
require 'thread'
|
8
|
+
|
9
|
+
|
10
|
+
class LogStash::Filters::IP2Location < LogStash::Filters::Base
|
11
|
+
config_name "ip2location"
|
12
|
+
|
13
|
+
# The path to the IP2Location.BIN database file which Logstash should use.
|
14
|
+
# If not specified, this will default to the IP2LOCATION-LITE-DB1.IPV6.BIN database that embedded in the plugin.
|
15
|
+
config :database, :validate => :path
|
16
|
+
|
17
|
+
# The field containing the IP address.
|
18
|
+
# If this field is an array, only the first value will be used.
|
19
|
+
config :source, :validate => :string, :required => true
|
20
|
+
|
21
|
+
# The field used to define iplocation as target.
|
22
|
+
config :target, :validate => :string, :default => 'ip2location'
|
23
|
+
|
24
|
+
# The field used to allow user to enable the use of memory mapped file.
|
25
|
+
config :use_memory_mapped, :validate => :boolean, :default => false
|
26
|
+
|
27
|
+
# The field used to define the size of the cache. It is not required and the default value is 10 000
|
28
|
+
config :cache_size, :validate => :number, :required => false, :default => 10_000
|
29
|
+
|
30
|
+
public
|
31
|
+
def register
|
32
|
+
if @database.nil?
|
33
|
+
@database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"IP2LOCATION-LITE-DB1.IPV6.BIN")).first
|
34
|
+
|
35
|
+
if @database.nil? || !File.exists?(@database)
|
36
|
+
raise "You must specify 'database => ...' in your ip2location filter (I looked for '#{@database}')"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
@logger.info("Using ip2location database", :path => @database)
|
41
|
+
|
42
|
+
@ip2locationfilter = org.logstash.filters.IP2LocationFilter.new(@source, @target, @database, @use_memory_mapped)
|
43
|
+
end
|
44
|
+
|
45
|
+
public
|
46
|
+
def filter(event)
|
47
|
+
ip = event.get(@source)
|
48
|
+
|
49
|
+
return unless filter?(event)
|
50
|
+
if value = Cache.find(event, ip, @ip2locationfilter, @cache_size).get('ip2location')
|
51
|
+
event.set('ip2location', value)
|
52
|
+
filter_matched(event)
|
53
|
+
else
|
54
|
+
tag_iplookup_unsuccessful(event)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def tag_iplookup_unsuccessful(event)
|
59
|
+
@logger.debug? && @logger.debug("IP #{event.get(@source)} was not found in the database", :event => event)
|
60
|
+
end
|
61
|
+
|
62
|
+
end # class LogStash::Filters::IP2Location
|
63
|
+
|
64
|
+
class OrderedHash
|
65
|
+
ONE = 1
|
66
|
+
|
67
|
+
attr_reader :times_queried # ip -> times queried
|
68
|
+
attr_reader :hash
|
69
|
+
|
70
|
+
def initialize
|
71
|
+
@times_queried = Hash.new(0) # ip -> times queried
|
72
|
+
@hash = {} # number of hits -> array of ips
|
73
|
+
end
|
74
|
+
|
75
|
+
def add(key)
|
76
|
+
hash[ONE] ||= []
|
77
|
+
hash[ONE] << key
|
78
|
+
times_queried[key] = ONE
|
79
|
+
end
|
80
|
+
|
81
|
+
def reorder(key)
|
82
|
+
number_of_queries = times_queried[key]
|
83
|
+
|
84
|
+
hash[number_of_queries].delete(key)
|
85
|
+
hash.delete(number_of_queries) if hash[number_of_queries].empty?
|
86
|
+
|
87
|
+
hash[number_of_queries + 1] ||= []
|
88
|
+
hash[number_of_queries + 1] << key
|
89
|
+
end
|
90
|
+
|
91
|
+
def increment(key)
|
92
|
+
add(key) unless times_queried.has_key?(key)
|
93
|
+
reorder(key)
|
94
|
+
times_queried[key] += 1
|
95
|
+
end
|
96
|
+
|
97
|
+
def delete_least_used
|
98
|
+
first_pile_with_someting.shift.tap { |key| times_queried.delete(key) }
|
99
|
+
end
|
100
|
+
|
101
|
+
def first_pile_with_someting
|
102
|
+
hash[hash.keys.min]
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
class Cache
|
107
|
+
ONE_DAY_IN_SECONDS = 86_400
|
108
|
+
|
109
|
+
@cache = {} # ip -> event
|
110
|
+
@timestamps = {} # ip -> time of caching
|
111
|
+
@times_queried = OrderedHash.new # ip -> times queried
|
112
|
+
@mutex = Mutex.new
|
113
|
+
|
114
|
+
class << self
|
115
|
+
attr_reader :cache
|
116
|
+
attr_reader :timestamps
|
117
|
+
attr_reader :times_queried
|
118
|
+
|
119
|
+
|
120
|
+
def find(event, ip, filter, cache_size)
|
121
|
+
synchronize do
|
122
|
+
if cache.has_key?(ip)
|
123
|
+
refresh_event(event, ip, filter) if too_old?(ip)
|
124
|
+
else
|
125
|
+
if cache_full?(cache_size)
|
126
|
+
make_room
|
127
|
+
end
|
128
|
+
cache_event(event, ip, filter)
|
129
|
+
end
|
130
|
+
times_queried.increment(ip)
|
131
|
+
cache[ip]
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def too_old?(ip)
|
136
|
+
timestamps[ip] < Time.now - ONE_DAY_IN_SECONDS
|
137
|
+
end
|
138
|
+
|
139
|
+
def make_room
|
140
|
+
key = times_queried.delete_least_used
|
141
|
+
cache.delete(key)
|
142
|
+
timestamps.delete(key)
|
143
|
+
end
|
144
|
+
|
145
|
+
def cache_full?(cache_size)
|
146
|
+
cache.size >= cache_size
|
147
|
+
end
|
148
|
+
|
149
|
+
def cache_event(event, ip, filter)
|
150
|
+
filter.handleEvent(event)
|
151
|
+
cache[ip] = event
|
152
|
+
timestamps[ip] = Time.now
|
153
|
+
end
|
154
|
+
|
155
|
+
def synchronize(&block)
|
156
|
+
@mutex.synchronize(&block)
|
157
|
+
end
|
158
|
+
|
159
|
+
alias_method :refresh_event, :cache_event
|
160
|
+
end
|
161
|
+
end
|
@@ -1,25 +1,25 @@
|
|
1
|
-
Gem::Specification.new do |s|
|
2
|
-
|
3
|
-
s.name = 'logstash-filter-ip2location'
|
4
|
-
s.version = '2.1.
|
5
|
-
s.licenses = ['Apache License (2.0)']
|
6
|
-
s.summary = "Logstash filter IP2Location"
|
7
|
-
s.description = "IP2Location filter plugin for Logstash enables Logstash's users to add geolocation information such as country, state, city, latitude, longitude, ZIP code, time zone, ISP, domain name, connection speed, IDD code, area code, weather station code, weather station name, MNC, MCC, mobile brand, elevation and usage type by IP address."
|
8
|
-
s.authors = ["IP2Location"]
|
9
|
-
s.email = 'support@ip2location.com'
|
10
|
-
s.homepage = "https://www.ip2location.com"
|
11
|
-
s.require_paths = ["lib", "vendor/jar-dependencies"]
|
12
|
-
|
13
|
-
# Files
|
14
|
-
s.files = Dir["lib/**/*",'spec/**/*',"vendor/**/*","vendor/jar-dependencies/**/*.jar","*.gemspec","*.md","Gemfile","LICENSE"]
|
15
|
-
|
16
|
-
# Tests
|
17
|
-
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
-
|
19
|
-
# Special flag to let us know this is actually a logstash plugin
|
20
|
-
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
21
|
-
|
22
|
-
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
24
|
-
s.add_development_dependency "logstash-devutils"
|
25
|
-
end
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
|
3
|
+
s.name = 'logstash-filter-ip2location'
|
4
|
+
s.version = '2.1.1'
|
5
|
+
s.licenses = ['Apache License (2.0)']
|
6
|
+
s.summary = "Logstash filter IP2Location"
|
7
|
+
s.description = "IP2Location filter plugin for Logstash enables Logstash's users to add geolocation information such as country, state, city, latitude, longitude, ZIP code, time zone, ISP, domain name, connection speed, IDD code, area code, weather station code, weather station name, MNC, MCC, mobile brand, elevation and usage type by IP address."
|
8
|
+
s.authors = ["IP2Location"]
|
9
|
+
s.email = 'support@ip2location.com'
|
10
|
+
s.homepage = "https://www.ip2location.com"
|
11
|
+
s.require_paths = ["lib", "vendor/jar-dependencies"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = Dir["lib/**/*",'spec/**/*',"vendor/**/*","vendor/jar-dependencies/**/*.jar","*.gemspec","*.md","Gemfile","LICENSE"]
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
24
|
+
s.add_development_dependency "logstash-devutils"
|
25
|
+
end
|
@@ -1,32 +1,32 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require_relative '../spec_helper'
|
3
|
-
require "logstash/filters/ip2location"
|
4
|
-
|
5
|
-
IP2LOCATIONDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/IP2LOCATION-LITE-
|
6
|
-
|
7
|
-
describe LogStash::Filters::IP2Location do
|
8
|
-
|
9
|
-
describe "normal test" do
|
10
|
-
config <<-CONFIG
|
11
|
-
filter {
|
12
|
-
ip2location {
|
13
|
-
source => "ip"
|
14
|
-
#database => "#{IP2LOCATIONDB}"
|
15
|
-
}
|
16
|
-
}
|
17
|
-
CONFIG
|
18
|
-
|
19
|
-
sample("ip" => "8.8.8.8") do
|
20
|
-
expect(subject.get("ip2location")).not_to be_empty
|
21
|
-
expect(subject.get("ip2location")["country_short"]).to eq("US")
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
sample("ip" => "2a01:04f8:0d16:26c2::") do
|
26
|
-
expect(subject.get("ip2location")).not_to be_empty
|
27
|
-
expect(subject.get("ip2location")["country_short"]).to eq("DE")
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../spec_helper'
|
3
|
+
require "logstash/filters/ip2location"
|
4
|
+
|
5
|
+
IP2LOCATIONDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/IP2LOCATION-LITE-DB1.IPV6.BIN").first
|
6
|
+
|
7
|
+
describe LogStash::Filters::IP2Location do
|
8
|
+
|
9
|
+
describe "normal test" do
|
10
|
+
config <<-CONFIG
|
11
|
+
filter {
|
12
|
+
ip2location {
|
13
|
+
source => "ip"
|
14
|
+
#database => "#{IP2LOCATIONDB}"
|
15
|
+
}
|
16
|
+
}
|
17
|
+
CONFIG
|
18
|
+
|
19
|
+
sample("ip" => "8.8.8.8") do
|
20
|
+
expect(subject.get("ip2location")).not_to be_empty
|
21
|
+
expect(subject.get("ip2location")["country_short"]).to eq("US")
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
sample("ip" => "2a01:04f8:0d16:26c2::") do
|
26
|
+
expect(subject.get("ip2location")).not_to be_empty
|
27
|
+
expect(subject.get("ip2location")["country_short"]).to eq("DE")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
32
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
Binary file
|
data/vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0.jar
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
File without changes
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-ip2location
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- IP2Location
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-11-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core-plugin-api
|
@@ -56,9 +56,13 @@ files:
|
|
56
56
|
- logstash-filter-ip2location.gemspec
|
57
57
|
- spec/filters/ip2location_spec.rb
|
58
58
|
- spec/spec_helper.rb
|
59
|
-
- vendor/IP2LOCATION-LITE-
|
60
|
-
- vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.
|
61
|
-
- vendor/jar-dependencies/
|
59
|
+
- vendor/IP2LOCATION-LITE-DB1.IPV6.BIN
|
60
|
+
- vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0.jar
|
61
|
+
- vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/META-INF/MANIFEST.MF
|
62
|
+
- vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/com/ip2location/IP2Location.class
|
63
|
+
- vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/com/ip2location/IPResult.class
|
64
|
+
- vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.2.0/ip2location-8.2.0/com/ip2location/MetaData.class
|
65
|
+
- vendor/jar-dependencies/org/logstash/filters/logstash-filter-ip2location/2.1.1/logstash-filter-ip2location-2.1.1.jar
|
62
66
|
homepage: https://www.ip2location.com
|
63
67
|
licenses:
|
64
68
|
- Apache License (2.0)
|
@@ -81,7 +85,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
81
85
|
- !ruby/object:Gem::Version
|
82
86
|
version: '0'
|
83
87
|
requirements: []
|
84
|
-
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 2.7.6.2
|
85
90
|
signing_key:
|
86
91
|
specification_version: 4
|
87
92
|
summary: Logstash filter IP2Location
|
Binary file
|