fluent-plugin-geoip-kk 1.0.2 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +117 -51
- data/Rakefile +1 -0
- data/fluent-plugin-geoip-kk.gemspec +18 -2
- data/lib/fluent/plugin/filter_geoip.rb +168 -38
- data/vendor/data/GeoLite2-City.mmdb.gz +0 -0
- metadata +6 -12
- data/.github/workflows/gem-push.yml +0 -34
- data/.gitignore +0 -22
- data/data/GeoLite2-City.mmdb +0 -0
- data/push.rb +0 -3
- data/test/bench_geoip_filter.rb +0 -50
- data/test/helper.rb +0 -36
- data/test.rb +0 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca6935a0106345ce2856c8f459b8b51b479f2058c07d06a054e65098b656ce8d
|
4
|
+
data.tar.gz: aa54bc540919b711a437803a1ee68c57d1457d315dc51448d3acaabf24197c8f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be6b1caf1cdb676909b21b19cd5827a232163844f8610f7f4490c40f442e2076ef9e4b40d3ac953d6f734cb88654223265eccf9a16def980342280599c1d4f17
|
7
|
+
data.tar.gz: 1503fab319f62797e5d1fede06acbdbb932b0003ddd6722628f259f391d0bc428d31d1662a5deb6d56c8720eb77df25c240803f3906891069633a427aaaf4760
|
data/README.md
CHANGED
@@ -9,9 +9,17 @@
|
|
9
9
|
|
10
10
|
| fluent-plugin-geoip-kk | fluentd | ruby |
|
11
11
|
|----------------------------|------------|--------|
|
12
|
-
| >= 1.0.
|
12
|
+
| >= 1.0.5 | >= v0.14.0 | >= 2.1 |
|
13
13
|
| < 1.0.0 | >= v0.12.0 | >= 1.9 |
|
14
14
|
|
15
|
+
## Features
|
16
|
+
|
17
|
+
- Automatic database handling (compressed/uncompressed)
|
18
|
+
- Memory-optimized database loading
|
19
|
+
- LRU caching for high-performance lookups
|
20
|
+
- Support for both legacy GeoIP and GeoIP2 databases
|
21
|
+
- Private IP address filtering
|
22
|
+
- Flexible output formatting (nested/flattened)
|
15
23
|
|
16
24
|
## Installation
|
17
25
|
|
@@ -23,78 +31,142 @@ $ gem install fluent-plugin-geoip-kk
|
|
23
31
|
$ sudo td-agent-gem install fluent-plugin-geoip-kk
|
24
32
|
```
|
25
33
|
|
34
|
+
## Configuration Parameters
|
35
|
+
|
36
|
+
| Parameter | Type | Default | Description |
|
37
|
+
|-----------|------|---------|-------------|
|
38
|
+
| key_name | string | client_ip | Field name that contains the IP address |
|
39
|
+
| out_key | string | geo | Output field name to store GeoIP data |
|
40
|
+
| database_path | string | auto-detect | Path to the GeoIP database file (supports .mmdb and .mmdb.gz) |
|
41
|
+
| flatten | bool | false | Flatten the GeoIP data structure |
|
42
|
+
| cache_size | integer | 8192 | Size of the LRU cache |
|
43
|
+
| cache_ttl | integer | 3600 | TTL for cached items in seconds |
|
44
|
+
| skip_private_ip | bool | true | Skip adding GeoIP data for private IP addresses |
|
45
|
+
| memory_cache | bool | true | Keep database in memory for better performance |
|
26
46
|
|
27
|
-
## Usage
|
47
|
+
## Usage Examples
|
28
48
|
|
29
|
-
###
|
49
|
+
### Basic Configuration
|
30
50
|
|
31
51
|
```xml
|
32
52
|
<filter access.nginx.**>
|
33
53
|
@type geoip
|
34
|
-
|
35
|
-
|
36
|
-
# out_key geo
|
54
|
+
key_name client_ip
|
55
|
+
out_key geo
|
37
56
|
</filter>
|
38
57
|
```
|
39
58
|
|
40
|
-
|
59
|
+
### Advanced Configuration
|
41
60
|
|
42
|
-
```
|
43
|
-
access.nginx
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
61
|
+
```xml
|
62
|
+
<filter access.nginx.**>
|
63
|
+
@type geoip
|
64
|
+
|
65
|
+
# IP address field configuration
|
66
|
+
key_name client_ip
|
67
|
+
out_key geo
|
68
|
+
|
69
|
+
# Database configuration
|
70
|
+
database_path /path/to/your/GeoLite2-City.mmdb.gz # Optional: supports both .mmdb and .mmdb.gz
|
71
|
+
|
72
|
+
# Performance optimization
|
73
|
+
memory_cache true # Keep database in memory (recommended)
|
74
|
+
cache_size 10000 # LRU cache size
|
75
|
+
cache_ttl 3600 # Cache TTL in seconds
|
76
|
+
|
77
|
+
# Output configuration
|
78
|
+
flatten false # Keep nested structure
|
79
|
+
|
80
|
+
# IP processing configuration
|
81
|
+
skip_private_ip true # Skip private IP addresses
|
82
|
+
</filter>
|
49
83
|
```
|
50
84
|
|
51
|
-
|
85
|
+
### Output Examples
|
52
86
|
|
87
|
+
#### Default Structure (flatten: false)
|
53
88
|
```json
|
54
|
-
|
55
|
-
"client_ip":"93.184.216.34",
|
56
|
-
"scheme":"http", "method":"GET", "host":"example.com",
|
57
|
-
"path":"/", "query":"-", "req_bytes":200, "referer":"-",
|
58
|
-
"status":200, "res_bytes":800, "res_body_bytes":600, "taken_time":0.001, "user_agent":"Mozilla/5.0",
|
89
|
+
{
|
90
|
+
"client_ip": "93.184.216.34",
|
59
91
|
"geo": {
|
60
|
-
"coordinates":
|
61
|
-
|
62
|
-
|
63
|
-
|
92
|
+
"coordinates": {
|
93
|
+
"latitude": 42.150800000000004,
|
94
|
+
"longitude": -70.8228,
|
95
|
+
"accuracy_radius": 100
|
96
|
+
},
|
97
|
+
"country": {
|
98
|
+
"code": "US",
|
99
|
+
"name": "United States"
|
100
|
+
},
|
101
|
+
"city": {
|
102
|
+
"name": "Norwell",
|
103
|
+
"confidence": 90
|
104
|
+
},
|
105
|
+
"region": {
|
106
|
+
"code": "MA",
|
107
|
+
"name": "Massachusetts"
|
108
|
+
},
|
109
|
+
"postal": {
|
110
|
+
"code": "02061",
|
111
|
+
"confidence": 95
|
112
|
+
},
|
113
|
+
"timezone": "America/New_York"
|
64
114
|
}
|
65
115
|
}
|
66
116
|
```
|
67
117
|
|
118
|
+
#### Flattened Structure (flatten: true)
|
119
|
+
```json
|
120
|
+
{
|
121
|
+
"client_ip": "93.184.216.34",
|
122
|
+
"geo_coordinates_latitude": 42.150800000000004,
|
123
|
+
"geo_coordinates_longitude": -70.8228,
|
124
|
+
"geo_coordinates_accuracy_radius": 100,
|
125
|
+
"geo_country_code": "US",
|
126
|
+
"geo_country_name": "United States",
|
127
|
+
"geo_city_name": "Norwell",
|
128
|
+
"geo_city_confidence": 90,
|
129
|
+
"geo_region_code": "MA",
|
130
|
+
"geo_region_name": "Massachusetts",
|
131
|
+
"geo_postal_code": "02061",
|
132
|
+
"geo_postal_confidence": 95,
|
133
|
+
"geo_timezone": "America/New_York"
|
134
|
+
}
|
135
|
+
```
|
68
136
|
|
69
|
-
##
|
70
|
-
- key_name *field_key*
|
71
|
-
|
72
|
-
Target key name. default client_ip.
|
73
|
-
|
74
|
-
- out_key *string*
|
137
|
+
## Performance Optimization
|
75
138
|
|
76
|
-
|
139
|
+
### Memory Usage vs Performance
|
77
140
|
|
78
|
-
|
141
|
+
The plugin offers two modes for database handling:
|
79
142
|
|
80
|
-
|
81
|
-
|
143
|
+
1. Memory Mode (Default, Recommended)
|
144
|
+
- Loads entire database into memory
|
145
|
+
- Fastest lookup performance
|
146
|
+
- Higher memory usage (~56MB for GeoLite2-City)
|
147
|
+
- Best for production environments
|
82
148
|
|
83
|
-
|
84
|
-
|
149
|
+
2. File Mode
|
150
|
+
- Keeps database on disk
|
151
|
+
- Lower memory usage
|
152
|
+
- Slightly slower lookups due to disk I/O
|
153
|
+
- Suitable for memory-constrained environments
|
85
154
|
|
155
|
+
### Caching Strategy
|
86
156
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
157
|
+
- LRU (Least Recently Used) cache with TTL
|
158
|
+
- Default cache size: 8192 entries
|
159
|
+
- Default TTL: 3600 seconds (1 hour)
|
160
|
+
- Adjust based on your traffic patterns:
|
161
|
+
- High unique IPs: Increase cache_size
|
162
|
+
- Stable IP patterns: Increase cache_ttl
|
91
163
|
|
92
|
-
|
93
|
-
* Not require to install Development Tools and geoip-dev library.
|
94
|
-
* ( fluent-plugin-geoip use geoip-c gem but our plugin use geoip. It's conflict. )
|
95
|
-
* 5-10 times faster by the LRU cache.
|
96
|
-
* See [benchmark](test/bench_geoip_filter.rb).
|
164
|
+
### Database Compression
|
97
165
|
|
166
|
+
- Database is distributed in compressed format (.mmdb.gz)
|
167
|
+
- Automatic handling of compressed/uncompressed files
|
168
|
+
- ~52% size reduction (56MB → 27MB)
|
169
|
+
- No performance impact when using memory_cache
|
98
170
|
|
99
171
|
## Development
|
100
172
|
|
@@ -127,12 +199,6 @@ This gem uses GitHub Actions for automated publishing. To publish a new version:
|
|
127
199
|
4. Push to the branch (`git push origin my-new-feature`)
|
128
200
|
5. Create a new Pull Request
|
129
201
|
|
130
|
-
|
131
|
-
## Copyright
|
132
|
-
|
133
|
-
Copyright (c) 2015 Yuri Umezaki
|
134
|
-
|
135
|
-
|
136
202
|
## License
|
137
203
|
|
138
204
|
[Apache License, Version 2.0.](http://www.apache.org/licenses/LICENSE-2.0)
|
data/Rakefile
CHANGED
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'fluent-plugin-geoip-kk'
|
8
|
-
spec.version = '1.0.
|
8
|
+
spec.version = '1.0.5'
|
9
9
|
spec.authors = ['kevin197011']
|
10
10
|
spec.email = ['kevin197011@outlook.com']
|
11
11
|
spec.homepage = 'https://github.com/kevin197011/fluent-plugin-geoip-kk'
|
@@ -13,11 +13,27 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.description = 'A Fluentd filter plugin that adds GeoIP information to records. Supports both legacy GeoIP and GeoIP2 databases, with LRU caching for improved performance.'
|
14
14
|
spec.license = 'Apache-2.0'
|
15
15
|
|
16
|
-
|
16
|
+
# Include all git tracked files and vendor/data directory
|
17
|
+
spec.files = Dir[
|
18
|
+
'lib/**/*',
|
19
|
+
'vendor/**/*.mmdb.gz',
|
20
|
+
'Gemfile',
|
21
|
+
'LICENSE',
|
22
|
+
'README.md',
|
23
|
+
'Rakefile',
|
24
|
+
'*.gemspec'
|
25
|
+
]
|
26
|
+
|
17
27
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
28
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
29
|
spec.require_paths = ['lib']
|
20
30
|
|
31
|
+
# Ensure vendor/data directory is included
|
32
|
+
spec.metadata = {
|
33
|
+
'data_dir' => 'vendor/data',
|
34
|
+
'database_file' => 'GeoLite2-City.mmdb.gz'
|
35
|
+
}
|
36
|
+
|
21
37
|
spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
22
38
|
spec.add_runtime_dependency 'geoip', '~> 0.1', '>= 0.1.22'
|
23
39
|
spec.add_runtime_dependency 'lru_redux', '~> 1.0', '>= 1.0.0'
|
@@ -4,6 +4,8 @@ require 'maxminddb'
|
|
4
4
|
require 'lru_redux'
|
5
5
|
require 'fluent/plugin/filter'
|
6
6
|
require 'ipaddr'
|
7
|
+
require 'zlib'
|
8
|
+
require 'stringio'
|
7
9
|
|
8
10
|
module Fluent
|
9
11
|
module Plugin
|
@@ -15,9 +17,10 @@ module Fluent
|
|
15
17
|
# Default cache size (number of IP addresses to cache)
|
16
18
|
DEFAULT_CACHE_SIZE = 8192
|
17
19
|
DEFAULT_CACHE_TTL = 3600 # 1 hour in seconds
|
20
|
+
DEFAULT_DB_FILENAME = 'GeoLite2-City.mmdb'
|
18
21
|
|
19
22
|
desc 'Path to the GeoIP database file'
|
20
|
-
config_param :database_path, :string, default:
|
23
|
+
config_param :database_path, :string, default: nil
|
21
24
|
|
22
25
|
desc 'Field name that contains the IP address'
|
23
26
|
config_param :key_name, :string, default: 'client_ip'
|
@@ -37,22 +40,37 @@ module Fluent
|
|
37
40
|
desc 'Skip adding GeoIP data for private IP addresses'
|
38
41
|
config_param :skip_private_ip, :bool, default: true
|
39
42
|
|
43
|
+
desc 'Keep database in memory'
|
44
|
+
config_param :memory_cache, :bool, default: true
|
45
|
+
|
40
46
|
def initialize
|
41
47
|
super
|
42
48
|
@geoip = nil
|
43
49
|
@ip_accessor = nil
|
50
|
+
@database_data = nil
|
44
51
|
end
|
45
52
|
|
46
53
|
def configure(conf)
|
47
54
|
super
|
48
55
|
|
49
|
-
|
50
|
-
|
51
|
-
end
|
56
|
+
start_time = Time.now
|
57
|
+
@database_path = resolve_database_path
|
52
58
|
|
53
59
|
# Initialize MaxMindDB
|
54
60
|
begin
|
55
|
-
|
61
|
+
if @memory_cache
|
62
|
+
@database_data = load_database(@database_path)
|
63
|
+
@geoip = MaxMindDB.new(StringIO.new(@database_data))
|
64
|
+
else
|
65
|
+
@geoip = MaxMindDB.new(decompress_if_needed(@database_path))
|
66
|
+
end
|
67
|
+
|
68
|
+
load_time = Time.now - start_time
|
69
|
+
log.info 'Loaded GeoIP database',
|
70
|
+
database: @database_path,
|
71
|
+
size: format_size(@database_data ? @database_data.size : File.size(@database_path)),
|
72
|
+
compressed_size: format_size(File.size(@database_path)),
|
73
|
+
load_time: format_time(load_time)
|
56
74
|
rescue StandardError => e
|
57
75
|
raise Fluent::ConfigError, "Failed to load GeoIP database: #{e.message}"
|
58
76
|
end
|
@@ -63,7 +81,16 @@ module Fluent
|
|
63
81
|
# Initialize cache with TTL support
|
64
82
|
@geoip_cache = LruRedux::TTL::Cache.new(@cache_size, @cache_ttl)
|
65
83
|
|
66
|
-
log.info
|
84
|
+
log.info 'Initialized GeoIP filter',
|
85
|
+
database: @database_path,
|
86
|
+
cache_size: @cache_size,
|
87
|
+
cache_ttl: @cache_ttl,
|
88
|
+
memory_cache: @memory_cache
|
89
|
+
end
|
90
|
+
|
91
|
+
def shutdown
|
92
|
+
super
|
93
|
+
@temp_db.unlink if @temp_db && File.exist?(@temp_db.path)
|
67
94
|
end
|
68
95
|
|
69
96
|
def filter(tag, time, record)
|
@@ -74,7 +101,7 @@ module Fluent
|
|
74
101
|
ip = IPAddr.new(ip_addr)
|
75
102
|
return record if @skip_private_ip && ip.private?
|
76
103
|
rescue IPAddr::InvalidAddressError => e
|
77
|
-
log.debug
|
104
|
+
log.debug 'Invalid IP address', ip: ip_addr, error: e.message
|
78
105
|
return record
|
79
106
|
end
|
80
107
|
|
@@ -89,71 +116,174 @@ module Fluent
|
|
89
116
|
|
90
117
|
record
|
91
118
|
rescue StandardError => e
|
92
|
-
log.error
|
119
|
+
log.error 'Failed to process GeoIP lookup', error_class: e.class, error: e.message, tag: tag, time: time
|
93
120
|
record
|
94
121
|
end
|
95
122
|
|
96
123
|
private
|
97
124
|
|
125
|
+
def resolve_database_path
|
126
|
+
return database_path if database_path && !database_path.empty?
|
127
|
+
|
128
|
+
# Search paths in order of preference
|
129
|
+
search_paths = [
|
130
|
+
# 1. Current directory
|
131
|
+
File.join(Dir.pwd, 'vendor/data', DEFAULT_DB_FILENAME),
|
132
|
+
File.join(Dir.pwd, 'vendor/data', "#{DEFAULT_DB_FILENAME}.gz"),
|
133
|
+
# 2. Gem vendor directory
|
134
|
+
File.expand_path("../../../vendor/data/#{DEFAULT_DB_FILENAME}", __FILE__),
|
135
|
+
File.expand_path("../../../vendor/data/#{DEFAULT_DB_FILENAME}.gz", __FILE__),
|
136
|
+
# 3. System-wide locations
|
137
|
+
'/usr/share/GeoIP/GeoLite2-City.mmdb',
|
138
|
+
'/usr/local/share/GeoIP/GeoLite2-City.mmdb'
|
139
|
+
]
|
140
|
+
|
141
|
+
# Find first existing database file
|
142
|
+
found_path = search_paths.find { |path| File.exist?(path) }
|
143
|
+
|
144
|
+
unless found_path
|
145
|
+
available_paths = search_paths.join("\n ")
|
146
|
+
raise Fluent::ConfigError, "Could not find GeoIP database file. Searched in:\n #{available_paths}"
|
147
|
+
end
|
148
|
+
|
149
|
+
log.info 'Found GeoIP database', path: found_path
|
150
|
+
found_path
|
151
|
+
end
|
152
|
+
|
153
|
+
def load_database(path)
|
154
|
+
if path.end_with?('.gz')
|
155
|
+
Zlib::GzipReader.open(path, &:read)
|
156
|
+
else
|
157
|
+
File.binread(path)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def decompress_if_needed(path)
|
162
|
+
return path unless path.end_with?('.gz')
|
163
|
+
|
164
|
+
decompressed_path = path.chomp('.gz')
|
165
|
+
unless File.exist?(decompressed_path)
|
166
|
+
File.open(decompressed_path, 'wb') do |file|
|
167
|
+
Zlib::GzipReader.open(path) do |gz|
|
168
|
+
file.write(gz.read)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
decompressed_path
|
173
|
+
end
|
174
|
+
|
175
|
+
def format_size(bytes)
|
176
|
+
units = %w[B KB MB GB]
|
177
|
+
size = bytes.to_f
|
178
|
+
unit_index = 0
|
179
|
+
|
180
|
+
while size > 1024 && unit_index < units.length - 1
|
181
|
+
size /= 1024
|
182
|
+
unit_index += 1
|
183
|
+
end
|
184
|
+
|
185
|
+
format('%.2f %s', size, units[unit_index])
|
186
|
+
end
|
187
|
+
|
188
|
+
def format_time(seconds)
|
189
|
+
if seconds < 1
|
190
|
+
format('%.2f ms', seconds * 1000)
|
191
|
+
else
|
192
|
+
format('%.2f s', seconds)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
98
196
|
def get_geoip(ip_addr)
|
99
197
|
geo_ip = @geoip.lookup(ip_addr)
|
100
198
|
return {} if geo_ip.nil?
|
101
199
|
|
102
|
-
{
|
103
|
-
|
104
|
-
|
105
|
-
'
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
200
|
+
result = {}
|
201
|
+
|
202
|
+
if (coordinates = get_coordinates(geo_ip))
|
203
|
+
result['coordinates'] = coordinates
|
204
|
+
end
|
205
|
+
|
206
|
+
if (country = get_country_info(geo_ip))
|
207
|
+
result['country'] = country
|
208
|
+
end
|
209
|
+
|
210
|
+
if (city = get_city_info(geo_ip))
|
211
|
+
result['city'] = city
|
212
|
+
end
|
213
|
+
|
214
|
+
if (region = get_region_info(geo_ip))
|
215
|
+
result['region'] = region
|
216
|
+
end
|
217
|
+
|
218
|
+
if (postal = get_postal_info(geo_ip))
|
219
|
+
result['postal'] = postal
|
220
|
+
end
|
221
|
+
|
222
|
+
result['timezone'] = geo_ip.location.time_zone if geo_ip.location && geo_ip.location.time_zone
|
223
|
+
|
224
|
+
result
|
110
225
|
end
|
111
226
|
|
112
227
|
def get_coordinates(geo_ip)
|
113
228
|
return nil unless geo_ip.location
|
114
229
|
|
115
|
-
{
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
230
|
+
result = {}
|
231
|
+
location = geo_ip.location
|
232
|
+
|
233
|
+
result['latitude'] = location.latitude if location.latitude
|
234
|
+
result['longitude'] = location.longitude if location.longitude
|
235
|
+
result['accuracy_radius'] = location.accuracy_radius if location.accuracy_radius
|
236
|
+
|
237
|
+
result.empty? ? nil : result
|
120
238
|
end
|
121
239
|
|
122
240
|
def get_country_info(geo_ip)
|
123
241
|
return nil unless geo_ip.country
|
124
242
|
|
125
|
-
{
|
126
|
-
|
127
|
-
|
128
|
-
|
243
|
+
result = {}
|
244
|
+
country = geo_ip.country
|
245
|
+
|
246
|
+
result['code'] = country.iso_code if country.iso_code
|
247
|
+
result['name'] = country.name if country.name
|
248
|
+
|
249
|
+
result.empty? ? nil : result
|
129
250
|
end
|
130
251
|
|
131
252
|
def get_city_info(geo_ip)
|
132
253
|
return nil unless geo_ip.city
|
133
254
|
|
134
|
-
{
|
135
|
-
|
136
|
-
|
137
|
-
|
255
|
+
result = {}
|
256
|
+
city = geo_ip.city
|
257
|
+
|
258
|
+
result['name'] = city.name if city.name
|
259
|
+
result['confidence'] = city.confidence if city.confidence
|
260
|
+
|
261
|
+
result.empty? ? nil : result
|
138
262
|
end
|
139
263
|
|
140
264
|
def get_region_info(geo_ip)
|
141
|
-
return nil unless geo_ip.subdivisions
|
265
|
+
return nil unless geo_ip.subdivisions && !geo_ip.subdivisions.empty?
|
142
266
|
|
143
267
|
subdivision = geo_ip.subdivisions.first
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
268
|
+
return nil unless subdivision
|
269
|
+
|
270
|
+
result = {}
|
271
|
+
result['code'] = subdivision.iso_code if subdivision.iso_code
|
272
|
+
result['name'] = subdivision.name if subdivision.name
|
273
|
+
|
274
|
+
result.empty? ? nil : result
|
148
275
|
end
|
149
276
|
|
150
277
|
def get_postal_info(geo_ip)
|
151
278
|
return nil unless geo_ip.postal
|
152
279
|
|
153
|
-
{
|
154
|
-
|
155
|
-
|
156
|
-
|
280
|
+
result = {}
|
281
|
+
postal = geo_ip.postal
|
282
|
+
|
283
|
+
result['code'] = postal.code if postal.code
|
284
|
+
result['confidence'] = postal.confidence if postal.confidence
|
285
|
+
|
286
|
+
result.empty? ? nil : result
|
157
287
|
end
|
158
288
|
|
159
289
|
def hash_flatten(hash, keys = [])
|
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-geoip-kk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- kevin197011
|
@@ -140,23 +140,19 @@ executables: []
|
|
140
140
|
extensions: []
|
141
141
|
extra_rdoc_files: []
|
142
142
|
files:
|
143
|
-
- ".github/workflows/gem-push.yml"
|
144
|
-
- ".gitignore"
|
145
143
|
- Gemfile
|
146
144
|
- LICENSE
|
147
145
|
- README.md
|
148
146
|
- Rakefile
|
149
|
-
- data/GeoLite2-City.mmdb
|
150
147
|
- fluent-plugin-geoip-kk.gemspec
|
151
148
|
- lib/fluent/plugin/filter_geoip.rb
|
152
|
-
-
|
153
|
-
- test.rb
|
154
|
-
- test/bench_geoip_filter.rb
|
155
|
-
- test/helper.rb
|
149
|
+
- vendor/data/GeoLite2-City.mmdb.gz
|
156
150
|
homepage: https://github.com/kevin197011/fluent-plugin-geoip-kk
|
157
151
|
licenses:
|
158
152
|
- Apache-2.0
|
159
|
-
metadata:
|
153
|
+
metadata:
|
154
|
+
data_dir: vendor/data
|
155
|
+
database_file: GeoLite2-City.mmdb.gz
|
160
156
|
post_install_message:
|
161
157
|
rdoc_options: []
|
162
158
|
require_paths:
|
@@ -176,6 +172,4 @@ rubygems_version: 3.4.19
|
|
176
172
|
signing_key:
|
177
173
|
specification_version: 4
|
178
174
|
summary: Fluentd filter plugin to add geoip
|
179
|
-
test_files:
|
180
|
-
- test/bench_geoip_filter.rb
|
181
|
-
- test/helper.rb
|
175
|
+
test_files: []
|
@@ -1,34 +0,0 @@
|
|
1
|
-
name: Ruby Gem
|
2
|
-
|
3
|
-
on:
|
4
|
-
push:
|
5
|
-
branches: [ "main" ]
|
6
|
-
pull_request:
|
7
|
-
branches: [ "main" ]
|
8
|
-
|
9
|
-
jobs:
|
10
|
-
build:
|
11
|
-
name: Build + Publish
|
12
|
-
runs-on: ubuntu-latest
|
13
|
-
permissions:
|
14
|
-
contents: read
|
15
|
-
packages: write
|
16
|
-
|
17
|
-
steps:
|
18
|
-
- uses: actions/checkout@v4
|
19
|
-
|
20
|
-
- name: Set up Ruby
|
21
|
-
uses: ruby/setup-ruby@v1
|
22
|
-
with:
|
23
|
-
ruby-version: '3.2' # 你可以指定 3.1、3.2、3.3 等
|
24
|
-
|
25
|
-
- name: Publish to RubyGems
|
26
|
-
run: |
|
27
|
-
mkdir -p $HOME/.gem
|
28
|
-
touch $HOME/.gem/credentials
|
29
|
-
chmod 0600 $HOME/.gem/credentials
|
30
|
-
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
31
|
-
gem build *.gemspec
|
32
|
-
gem push *.gem
|
33
|
-
env:
|
34
|
-
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}"
|
data/.gitignore
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
*.gem
|
2
|
-
*.rbc
|
3
|
-
.bundle
|
4
|
-
.config
|
5
|
-
.yardoc
|
6
|
-
Gemfile.lock
|
7
|
-
InstalledFiles
|
8
|
-
_yardoc
|
9
|
-
coverage
|
10
|
-
doc/
|
11
|
-
lib/bundler/man
|
12
|
-
pkg
|
13
|
-
rdoc
|
14
|
-
spec/reports
|
15
|
-
test/tmp
|
16
|
-
test/version_tmp
|
17
|
-
tmp
|
18
|
-
*.bundle
|
19
|
-
*.so
|
20
|
-
*.o
|
21
|
-
*.a
|
22
|
-
mkmf.log
|
data/data/GeoLite2-City.mmdb
DELETED
Binary file
|
data/push.rb
DELETED
data/test/bench_geoip_filter.rb
DELETED
@@ -1,50 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative 'helper'
|
4
|
-
require 'fluent/plugin/filter_geoip'
|
5
|
-
|
6
|
-
# setup
|
7
|
-
Fluent::Test.setup
|
8
|
-
config = %(
|
9
|
-
@type geoip
|
10
|
-
out_key geo
|
11
|
-
)
|
12
|
-
time = Time.now.to_i
|
13
|
-
tag = 'foo.bar'
|
14
|
-
driver = Fluent::Test::FilterTestDriver.new(Fluent::GeoipFilter, tag).configure(config, true)
|
15
|
-
|
16
|
-
# bench
|
17
|
-
require 'benchmark'
|
18
|
-
require 'ipaddr'
|
19
|
-
n = 100_000
|
20
|
-
Benchmark.bm(7) do |x|
|
21
|
-
# 8192 random IP
|
22
|
-
x.report do
|
23
|
-
driver.run do
|
24
|
-
n.times do
|
25
|
-
driver.emit({ 'client_ip' => IPAddr.new(rand(2**13) * 2**19, Socket::AF_INET).to_s }, time)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
# Without LRU cache
|
32
|
-
# user system total real
|
33
|
-
# 11.410000 2.730000 14.140000 ( 15.431248)
|
34
|
-
# With LRU cache(8192) & random 1024 IP
|
35
|
-
# user system total real
|
36
|
-
# 1.250000 0.070000 1.320000 ( 1.322339)
|
37
|
-
# With LRU cache(8192) & random 8192 IP
|
38
|
-
# user system total real
|
39
|
-
# 1.890000 0.210000 2.100000 ( 2.102445)
|
40
|
-
# With LRU cache(8192) & random 16384 IP
|
41
|
-
# user system total real
|
42
|
-
# 8.450000 2.070000 10.520000 ( 12.170379)
|
43
|
-
# With LRU cache(8192) & random 65536 IP
|
44
|
-
# user system total real
|
45
|
-
# 11.890000 2.820000 14.710000 ( 16.051674)
|
46
|
-
|
47
|
-
# fluent-plugin-geoip
|
48
|
-
# (https://github.com/y-ken/fluent-plugin-geoip)
|
49
|
-
# user system total real
|
50
|
-
# 11.540000 0.270000 11.810000 ( 12.685000)
|
data/test/helper.rb
DELETED
@@ -1,36 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
require 'bundler'
|
5
|
-
begin
|
6
|
-
Bundler.setup(:default, :development)
|
7
|
-
rescue Bundler::BundlerError => e
|
8
|
-
warn e.message
|
9
|
-
warn 'Run `bundle install` to install missing gems'
|
10
|
-
exit e.status_code
|
11
|
-
end
|
12
|
-
require 'test/unit'
|
13
|
-
|
14
|
-
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
15
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
16
|
-
require 'fluent/test'
|
17
|
-
unless ENV.key?('VERBOSE')
|
18
|
-
nulllogger = Object.new
|
19
|
-
nulllogger.instance_eval do |_obj|
|
20
|
-
def method_missing(method, *args)
|
21
|
-
# pass
|
22
|
-
end
|
23
|
-
end
|
24
|
-
$log = nulllogger
|
25
|
-
end
|
26
|
-
require 'fluent/test/helpers'
|
27
|
-
require 'fluent/test/driver/filter'
|
28
|
-
require 'fluent/plugin/filter_geoip'
|
29
|
-
|
30
|
-
module Test
|
31
|
-
module Unit
|
32
|
-
class TestCase
|
33
|
-
include Fluent::Test::Helpers
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
data/test.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'maxminddb'
|
4
|
-
|
5
|
-
db = MaxMindDB.new('data/GeoLite2-City.mmdb')
|
6
|
-
|
7
|
-
ip = '128.101.101.101'
|
8
|
-
|
9
|
-
begin
|
10
|
-
result = db.lookup(ip)
|
11
|
-
|
12
|
-
puts "IP: #{ip}"
|
13
|
-
puts "Country: #{result.country.name}"
|
14
|
-
puts "City: #{result.city.name}" if result.city.name
|
15
|
-
puts "Latitude: #{result.location.latitude}" if result.location.latitude
|
16
|
-
puts "Longitude: #{result.location.longitude}" if result.location.longitude
|
17
|
-
puts result.inspect
|
18
|
-
rescue MaxMindDB::AddressNotFoundError
|
19
|
-
puts 'IP address not found in the database.'
|
20
|
-
end
|