fluent-plugin-qqwry 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/Gemfile +6 -0
- data/LICENSE +14 -0
- data/README.md +150 -0
- data/Rakefile +9 -0
- data/data/qqwry.dat +0 -0
- data/fluent-plugin-qqwry.gemspec +24 -0
- data/lib/fluent/plugin/out_qqwry.rb +166 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_qqwry.rb +309 -0
- metadata +127 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 13b7b4e4a05362dd611e2ca9bd8d9cf1fb8fa5b5
|
4
|
+
data.tar.gz: 811fcc85eaa7dae22545904f30f4687a2c11a152
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 054d4542cb901f72524851855e6229cd52311580ab4cd5a9b75e88e1448ab4b11989f086239d6582061ef787d096179b0bce034e9b88f796d69dc1dea6de46e7
|
7
|
+
data.tar.gz: 7ca2d977db399a85c87c1531a74d62a4fe23e42e6f49c67cc046c63f6bacf5dbcca58ece36d834db44954c0186967034cf2df13a70ed3ccf6c7bdd7e2c5ba61c
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (c) 2013- Kentaro Yoshida
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/README.md
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
# fluent-plugin-qqwry
|
2
|
+
|
3
|
+
Fluentd Output plugin to add information about geographical location of IP addresses with QQWry databases.
|
4
|
+
|
5
|
+
fluent-plugin-geoip has bundled qqwry.dat
|
6
|
+
|
7
|
+
## Dependency
|
8
|
+
|
9
|
+
before use, install dependent library as:
|
10
|
+
|
11
|
+
## Installation
|
12
|
+
|
13
|
+
install with `gem` or `fluent-gem` command as:
|
14
|
+
|
15
|
+
```bash
|
16
|
+
# for fluentd
|
17
|
+
$ gem install fluent-plugin-qqwry
|
18
|
+
|
19
|
+
# for td-agent
|
20
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-qqwry
|
21
|
+
```
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
```xml
|
26
|
+
<match access.apache>
|
27
|
+
type qqwry
|
28
|
+
|
29
|
+
# Specify one or more qqwry lookup field which has ip address (default: host)
|
30
|
+
# in the case of accessing nested value, delimit keys by dot like 'host.ip'.
|
31
|
+
qqwry_lookup_key host
|
32
|
+
|
33
|
+
# Specify optional qqwry database (using bundled GeoLiteCity databse by default)
|
34
|
+
qqwry_database '/path/to/your/qqwry.dat'
|
35
|
+
|
36
|
+
# Set adding field with placeholder (more than one settings are required.)
|
37
|
+
<record>
|
38
|
+
city ${city['host']}
|
39
|
+
area ${area_code['host']}
|
40
|
+
country ${country['host']}
|
41
|
+
province ${province['host']}
|
42
|
+
</record>
|
43
|
+
|
44
|
+
# Settings for tag
|
45
|
+
remove_tag_prefix access.
|
46
|
+
tag qqwry.${tag}
|
47
|
+
|
48
|
+
# Set log_level for fluentd-v0.10.43 or earlier (default: warn)
|
49
|
+
log_level info
|
50
|
+
|
51
|
+
# Set buffering time (default: 0s)
|
52
|
+
flush_interval 1s
|
53
|
+
</match>
|
54
|
+
```
|
55
|
+
|
56
|
+
#### Tips: how to geolocate multiple key
|
57
|
+
|
58
|
+
```xml
|
59
|
+
<match access.apache>
|
60
|
+
type qqwry
|
61
|
+
qqwry_lookup_key user1_host, user2_host
|
62
|
+
<record>
|
63
|
+
user1_city ${city['user1_host']}
|
64
|
+
user2_city ${city['user2_host']}
|
65
|
+
</record>
|
66
|
+
remove_tag_prefix access.
|
67
|
+
tag qqwry.${tag}
|
68
|
+
</match>
|
69
|
+
```
|
70
|
+
|
71
|
+
## Tutorial
|
72
|
+
|
73
|
+
#### configuration
|
74
|
+
|
75
|
+
```xml
|
76
|
+
<source>
|
77
|
+
type forward
|
78
|
+
</source>
|
79
|
+
|
80
|
+
<match test.qqwry>
|
81
|
+
type copy
|
82
|
+
<store>
|
83
|
+
type stdout
|
84
|
+
</store>
|
85
|
+
<store>
|
86
|
+
type qqwry
|
87
|
+
qqwry_lookup_key host
|
88
|
+
<record>
|
89
|
+
city ${city['host']}
|
90
|
+
lat ${latitude['host']}
|
91
|
+
lon ${longitude['host']}
|
92
|
+
</record>
|
93
|
+
remove_tag_prefix test.
|
94
|
+
tag debug.${tag}
|
95
|
+
</store>
|
96
|
+
</match>
|
97
|
+
|
98
|
+
<match debug.**>
|
99
|
+
type stdout
|
100
|
+
</match>
|
101
|
+
```
|
102
|
+
|
103
|
+
#### result
|
104
|
+
|
105
|
+
```bash
|
106
|
+
# forward record with Google's ip address.
|
107
|
+
$ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.qqwry
|
108
|
+
|
109
|
+
# check the result at stdout
|
110
|
+
$ tail /var/log/td-agent/td-agent.log
|
111
|
+
2013-08-04 16:21:32 +0900 test.qqwry: {"host":"66.102.9.80","message":"test"}
|
112
|
+
2013-08-04 16:21:32 +0900 debug.qqwry: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
|
113
|
+
```
|
114
|
+
|
115
|
+
## Parameters
|
116
|
+
|
117
|
+
* `include_tag_key` (default: false)
|
118
|
+
* `tag_key`
|
119
|
+
|
120
|
+
Add original tag name into filtered record using SetTagKeyMixin.<br />
|
121
|
+
Further details are written at http://docs.fluentd.org/articles/in_exec
|
122
|
+
|
123
|
+
* `remove_tag_prefix`
|
124
|
+
* `remove_tag_suffix`
|
125
|
+
* `add_tag_prefix`
|
126
|
+
* `add_tag_suffix`
|
127
|
+
|
128
|
+
Set one or more option are required unless using `tag` option for editing tag name. (HandleTagNameMixin feature)
|
129
|
+
|
130
|
+
* `tag`
|
131
|
+
|
132
|
+
On using this option with tag placeholder like `tag qqwry.${tag}` (test code is available at [test_out_qqwry.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
|
133
|
+
|
134
|
+
* `flush_interval` (default: 0 sec)
|
135
|
+
|
136
|
+
## Contributing
|
137
|
+
|
138
|
+
1. Fork it
|
139
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
140
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
141
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
142
|
+
5. Create new Pull Request
|
143
|
+
|
144
|
+
## Copyright
|
145
|
+
|
146
|
+
Copyright (c) 2014- Chris Song ([@fakechris](http://weibo.com/songchris))
|
147
|
+
|
148
|
+
## License
|
149
|
+
|
150
|
+
Apache License, Version 2.0
|
data/Rakefile
ADDED
data/data/qqwry.dat
ADDED
Binary file
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "fluent-plugin-qqwry"
|
7
|
+
spec.version = "0.0.1"
|
8
|
+
spec.authors = ["Chris Song"]
|
9
|
+
spec.email = ["fakechris@gmail.com"]
|
10
|
+
spec.summary = %q{Fluentd Output plugin to add information about geographical location of IP addresses with QQWry databases.}
|
11
|
+
spec.homepage = "https://github.com/fakechris/fluent-plugin-qqwry"
|
12
|
+
spec.license = "Apache License, Version 2.0"
|
13
|
+
|
14
|
+
spec.files = `git ls-files`.split($/)
|
15
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
16
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
17
|
+
spec.require_paths = ["lib"]
|
18
|
+
|
19
|
+
spec.add_development_dependency "bundler"
|
20
|
+
spec.add_development_dependency "rake"
|
21
|
+
spec.add_runtime_dependency "fluentd"
|
22
|
+
spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
|
23
|
+
spec.add_runtime_dependency "qqwry"
|
24
|
+
end
|
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'fluent/mixin/rewrite_tag_name'
|
2
|
+
|
3
|
+
class Fluent::QQWryOutput < Fluent::BufferedOutput
|
4
|
+
Fluent::Plugin.register_output('qqwry', self)
|
5
|
+
|
6
|
+
REGEXP_JSON = /(^[\[\{].+[\]\}]$|^[\d\.\-]+$)/
|
7
|
+
REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<qqwry_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
|
8
|
+
REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
|
9
|
+
QQWRY_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
|
10
|
+
|
11
|
+
config_param :qqwry_database, :string, :default => File.dirname(__FILE__) + '/../../../data/qqwry.dat'
|
12
|
+
config_param :qqwry_lookup_key, :string, :default => 'host'
|
13
|
+
config_param :tag, :string, :default => nil
|
14
|
+
|
15
|
+
include Fluent::HandleTagNameMixin
|
16
|
+
include Fluent::SetTagKeyMixin
|
17
|
+
config_set_default :include_tag_key, false
|
18
|
+
|
19
|
+
include Fluent::Mixin::RewriteTagName
|
20
|
+
config_param :hostname_command, :string, :default => 'hostname'
|
21
|
+
|
22
|
+
config_param :flush_interval, :time, :default => 0
|
23
|
+
config_param :log_level, :string, :default => 'warn'
|
24
|
+
|
25
|
+
# Define `log` method for v0.10.42 or earlier
|
26
|
+
unless method_defined?(:log)
|
27
|
+
define_method("log") { $log }
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize
|
31
|
+
require 'qqwry'
|
32
|
+
|
33
|
+
super
|
34
|
+
end
|
35
|
+
|
36
|
+
def configure(conf)
|
37
|
+
super
|
38
|
+
|
39
|
+
@map = {}
|
40
|
+
@qqwry_lookup_key = @qqwry_lookup_key.split(/\s*,\s*/)
|
41
|
+
|
42
|
+
# enable_key_* format (legacy format)
|
43
|
+
conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
|
44
|
+
qqwry_key = key.sub('enable_key_','')
|
45
|
+
raise Fluent::ConfigError, "qqwry: unsupported key #{qqwry_key}" unless QQWRY_KEYS.include?(qqwry_key)
|
46
|
+
@qqwry_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
|
47
|
+
if record_key.nil?
|
48
|
+
raise Fluent::ConfigError, "qqwry: missing value found at '#{key} #{lookup_field}'"
|
49
|
+
end
|
50
|
+
@map.store(record_key, "${#{qqwry_key}['#{lookup_field}']}")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
|
54
|
+
log.warn "qqwry: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
|
55
|
+
log.warn "qqwry: for further details referable to https://github.com/fakechris/fluent-plugin-qqwry"
|
56
|
+
end
|
57
|
+
|
58
|
+
# <record></record> directive
|
59
|
+
conf.elements.select { |element| element.name == 'record' }.each { |element|
|
60
|
+
element.each_pair { |k, v|
|
61
|
+
element.has_key?(k) # to suppress unread configuration warning
|
62
|
+
@map[k] = v
|
63
|
+
validate_json = Proc.new {
|
64
|
+
begin
|
65
|
+
dummy_text = Yajl::Encoder.encode('dummy_text')
|
66
|
+
Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
|
67
|
+
rescue Yajl::ParseError => e
|
68
|
+
raise Fluent::ConfigError, "qqwry: failed to parse '#{v}' as json."
|
69
|
+
end
|
70
|
+
}
|
71
|
+
validate_json.call if v.match(REGEXP_JSON)
|
72
|
+
}
|
73
|
+
}
|
74
|
+
@placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
|
75
|
+
@placeholder_keys.each do |key|
|
76
|
+
qqwry_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:qqwry_key]
|
77
|
+
raise Fluent::ConfigError, "qqwry: unsupported key #{qqwry_key}" unless QQWRY_KEYS.include?(qqwry_key)
|
78
|
+
end
|
79
|
+
@placeholder_expander = PlaceholderExpander.new
|
80
|
+
|
81
|
+
if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
|
82
|
+
raise Fluent::ConfigError, "qqwry: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
|
83
|
+
end
|
84
|
+
|
85
|
+
@qqwry = QQWry::Database.new(@qqwry_database)
|
86
|
+
end
|
87
|
+
|
88
|
+
def start
|
89
|
+
super
|
90
|
+
end
|
91
|
+
|
92
|
+
def format(tag, time, record)
|
93
|
+
[tag, time, record].to_msgpack
|
94
|
+
end
|
95
|
+
|
96
|
+
def shutdown
|
97
|
+
super
|
98
|
+
end
|
99
|
+
|
100
|
+
def write(chunk)
|
101
|
+
chunk.msgpack_each do |tag, time, record|
|
102
|
+
Fluent::Engine.emit(tag, time, add_qqwry_field(record))
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
private
|
107
|
+
def add_qqwry_field(record)
|
108
|
+
placeholder = create_placeholder(geolocate(get_address(record)))
|
109
|
+
@map.each do |record_key, value|
|
110
|
+
if value.match(REGEXP_PLACEHOLDER_SINGLE)
|
111
|
+
rewrited = placeholder[value]
|
112
|
+
elsif value.match(REGEXP_JSON)
|
113
|
+
rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
|
114
|
+
Yajl::Encoder.encode(placeholder[match])
|
115
|
+
}
|
116
|
+
rewrited = parse_json(rewrited)
|
117
|
+
else
|
118
|
+
rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
|
119
|
+
end
|
120
|
+
record.store(record_key, rewrited)
|
121
|
+
end
|
122
|
+
return record
|
123
|
+
end
|
124
|
+
|
125
|
+
def parse_json(message)
|
126
|
+
begin
|
127
|
+
return Yajl::Parser.parse(message)
|
128
|
+
rescue Yajl::ParseError => e
|
129
|
+
log.info "qqwry: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
|
130
|
+
return nil
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def get_address(record)
|
135
|
+
address = {}
|
136
|
+
@qqwry_lookup_key.each do |field|
|
137
|
+
key = field.split('.')
|
138
|
+
obj = record
|
139
|
+
key.each {|k|
|
140
|
+
break obj = nil if not obj.has_key?(k)
|
141
|
+
obj = obj[k]
|
142
|
+
}
|
143
|
+
address.store(field, obj)
|
144
|
+
end
|
145
|
+
return address
|
146
|
+
end
|
147
|
+
|
148
|
+
def geolocate(addresses)
|
149
|
+
geodata = {}
|
150
|
+
addresses.each do |field, ip|
|
151
|
+
geo = ip.nil? ? nil : @qqwry.query(ip)
|
152
|
+
geodata.store(field, geo)
|
153
|
+
end
|
154
|
+
return geodata
|
155
|
+
end
|
156
|
+
|
157
|
+
def create_placeholder(geodata)
|
158
|
+
placeholder = {}
|
159
|
+
@placeholder_keys.each do |placeholder_key|
|
160
|
+
position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
|
161
|
+
next if position.nil? or geodata[position[:record_key]].nil?
|
162
|
+
placeholder.store(placeholder_key, geodata[position[:record_key]][position[:qqwry_key].to_sym])
|
163
|
+
end
|
164
|
+
return placeholder
|
165
|
+
end
|
166
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_geoip'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,309 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class QQWryOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
qqwry_lookup_key host
|
10
|
+
enable_key_city qqwry_city
|
11
|
+
remove_tag_prefix input.
|
12
|
+
tag qqwry.${tag}
|
13
|
+
]
|
14
|
+
|
15
|
+
def create_driver(conf=CONFIG,tag='test')
|
16
|
+
Fluent::Test::OutputTestDriver.new(Fluent::QQWryOutput, tag).configure(conf)
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_configure
|
20
|
+
assert_raise(Fluent::ConfigError) {
|
21
|
+
d = create_driver('')
|
22
|
+
}
|
23
|
+
assert_raise(Fluent::ConfigError) {
|
24
|
+
d = create_driver('enable_key_cities')
|
25
|
+
}
|
26
|
+
d = create_driver %[
|
27
|
+
enable_key_city qqwry_city
|
28
|
+
remove_tag_prefix input.
|
29
|
+
tag qqwry.${tag}
|
30
|
+
]
|
31
|
+
assert_equal 'qqwry_city', d.instance.config['enable_key_city']
|
32
|
+
|
33
|
+
# multiple key config
|
34
|
+
d = create_driver %[
|
35
|
+
qqwry_lookup_key from.ip, to.ip
|
36
|
+
enable_key_city from_city, to_city
|
37
|
+
remove_tag_prefix input.
|
38
|
+
tag qqwry.${tag}
|
39
|
+
]
|
40
|
+
assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
|
41
|
+
|
42
|
+
# multiple key config (bad configure)
|
43
|
+
assert_raise(Fluent::ConfigError) {
|
44
|
+
d = create_driver %[
|
45
|
+
qqwry_lookup_key from.ip, to.ip
|
46
|
+
enable_key_city from_city
|
47
|
+
enable_key_region from_region
|
48
|
+
remove_tag_prefix input.
|
49
|
+
tag qqwry.${tag}
|
50
|
+
]
|
51
|
+
}
|
52
|
+
|
53
|
+
# invalid json structure
|
54
|
+
assert_raise(Fluent::ConfigError) {
|
55
|
+
d = create_driver %[
|
56
|
+
qqwry_lookup_key host
|
57
|
+
<record>
|
58
|
+
invalid_json {"foo" => 123}
|
59
|
+
</record>
|
60
|
+
remove_tag_prefix input.
|
61
|
+
tag qqwry.${tag}
|
62
|
+
]
|
63
|
+
}
|
64
|
+
assert_raise(Fluent::ConfigError) {
|
65
|
+
d = create_driver %[
|
66
|
+
qqwry_lookup_key host
|
67
|
+
<record>
|
68
|
+
invalid_json {"foo" : string, "bar" : 123}
|
69
|
+
</record>
|
70
|
+
remove_tag_prefix input.
|
71
|
+
tag qqwry.${tag}
|
72
|
+
]
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_emit
|
77
|
+
d1 = create_driver(CONFIG, 'input.access')
|
78
|
+
d1.run do
|
79
|
+
d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
80
|
+
d1.emit({'message' => 'missing field'})
|
81
|
+
end
|
82
|
+
emits = d1.emits
|
83
|
+
assert_equal 2, emits.length
|
84
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
85
|
+
assert_equal 'Mountain View', emits[0][2]['qqwry_city']
|
86
|
+
assert_equal nil, emits[1][2]['qqwry_city']
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_emit_tag_option
|
90
|
+
d1 = create_driver(%[
|
91
|
+
qqwry_lookup_key host
|
92
|
+
<record>
|
93
|
+
qqwry_city ${city['host']}
|
94
|
+
</record>
|
95
|
+
remove_tag_prefix input.
|
96
|
+
tag qqwry.${tag}
|
97
|
+
], 'input.access')
|
98
|
+
d1.run do
|
99
|
+
d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
100
|
+
d1.emit({'message' => 'missing field'})
|
101
|
+
end
|
102
|
+
emits = d1.emits
|
103
|
+
assert_equal 2, emits.length
|
104
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
105
|
+
assert_equal 'Mountain View', emits[0][2]['qqwry_city']
|
106
|
+
assert_equal nil, emits[1][2]['qqwry_city']
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_emit_tag_parts
|
110
|
+
d1 = create_driver(%[
|
111
|
+
qqwry_lookup_key host
|
112
|
+
<record>
|
113
|
+
qqwry_city ${city['host']}
|
114
|
+
</record>
|
115
|
+
tag qqwry.${tag_parts[1]}.${tag_parts[2..3]}.${tag_parts[-1]}
|
116
|
+
], '0.1.2.3')
|
117
|
+
d1.run do
|
118
|
+
d1.emit({'host' => '66.102.3.80'})
|
119
|
+
end
|
120
|
+
emits = d1.emits
|
121
|
+
assert_equal 1, emits.length
|
122
|
+
assert_equal 'qqwry.1.2.3.3', emits[0][0] # tag
|
123
|
+
assert_equal 'Mountain View', emits[0][2]['qqwry_city']
|
124
|
+
end
|
125
|
+
|
126
|
+
def test_emit_nested_attr
|
127
|
+
d1 = create_driver(%[
|
128
|
+
qqwry_lookup_key host.ip
|
129
|
+
enable_key_city qqwry_city
|
130
|
+
remove_tag_prefix input.
|
131
|
+
add_tag_prefix qqwry.
|
132
|
+
], 'input.access')
|
133
|
+
d1.run do
|
134
|
+
d1.emit({'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'})
|
135
|
+
d1.emit({'message' => 'missing field'})
|
136
|
+
end
|
137
|
+
emits = d1.emits
|
138
|
+
assert_equal 2, emits.length
|
139
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
140
|
+
assert_equal 'Mountain View', emits[0][2]['qqwry_city']
|
141
|
+
assert_equal nil, emits[1][2]['qqwry_city']
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_emit_with_unknown_address
|
145
|
+
d1 = create_driver(CONFIG, 'input.access')
|
146
|
+
d1.run do
|
147
|
+
# 203.0.113.1 is a test address described in RFC5737
|
148
|
+
d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
149
|
+
d1.emit({'host' => '0', 'message' => 'invalid ip'})
|
150
|
+
end
|
151
|
+
emits = d1.emits
|
152
|
+
assert_equal 2, emits.length
|
153
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
154
|
+
assert_equal nil, emits[0][2]['qqwry_city']
|
155
|
+
assert_equal 'qqwry.access', emits[1][0] # tag
|
156
|
+
assert_equal nil, emits[1][2]['qqwry_city']
|
157
|
+
end
|
158
|
+
|
159
|
+
def test_emit_multiple_key
|
160
|
+
d1 = create_driver(%[
|
161
|
+
qqwry_lookup_key from.ip, to.ip
|
162
|
+
enable_key_city from_city, to_city
|
163
|
+
remove_tag_prefix input.
|
164
|
+
add_tag_prefix qqwry.
|
165
|
+
], 'input.access')
|
166
|
+
d1.run do
|
167
|
+
d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
|
168
|
+
d1.emit({'message' => 'missing field'})
|
169
|
+
end
|
170
|
+
emits = d1.emits
|
171
|
+
assert_equal 2, emits.length
|
172
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
173
|
+
assert_equal 'Mountain View', emits[0][2]['from_city']
|
174
|
+
assert_equal 'Musashino', emits[0][2]['to_city']
|
175
|
+
assert_equal nil, emits[1][2]['from_city']
|
176
|
+
assert_equal nil, emits[1][2]['to_city']
|
177
|
+
end
|
178
|
+
|
179
|
+
def test_emit_multiple_key_multiple_record
|
180
|
+
d1 = create_driver(%[
|
181
|
+
qqwry_lookup_key from.ip, to.ip
|
182
|
+
enable_key_city from_city, to_city
|
183
|
+
enable_key_country_name from_country, to_country
|
184
|
+
remove_tag_prefix input.
|
185
|
+
add_tag_prefix qqwry.
|
186
|
+
], 'input.access')
|
187
|
+
d1.run do
|
188
|
+
d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
|
189
|
+
d1.emit({'from' => {'ip' => '66.102.3.80'}})
|
190
|
+
d1.emit({'message' => 'missing field'})
|
191
|
+
end
|
192
|
+
emits = d1.emits
|
193
|
+
assert_equal 3, emits.length
|
194
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
195
|
+
assert_equal 'Mountain View', emits[0][2]['from_city']
|
196
|
+
assert_equal 'United States', emits[0][2]['from_country']
|
197
|
+
assert_equal 'Musashino', emits[0][2]['to_city']
|
198
|
+
assert_equal 'Japan', emits[0][2]['to_country']
|
199
|
+
|
200
|
+
assert_equal 'Mountain View', emits[1][2]['from_city']
|
201
|
+
assert_equal 'United States', emits[1][2]['from_country']
|
202
|
+
assert_equal nil, emits[1][2]['to_city']
|
203
|
+
assert_equal nil, emits[1][2]['to_country']
|
204
|
+
|
205
|
+
assert_equal nil, emits[2][2]['from_city']
|
206
|
+
assert_equal nil, emits[2][2]['from_country']
|
207
|
+
assert_equal nil, emits[2][2]['to_city']
|
208
|
+
assert_equal nil, emits[2][2]['to_country']
|
209
|
+
end
|
210
|
+
|
211
|
+
def test_emit_record_directive
|
212
|
+
d1 = create_driver(%[
|
213
|
+
qqwry_lookup_key from.ip
|
214
|
+
<record>
|
215
|
+
from_city ${city['from.ip']}
|
216
|
+
from_country ${country_name['from.ip']}
|
217
|
+
latitude ${latitude['from.ip']}
|
218
|
+
longitude ${longitude['from.ip']}
|
219
|
+
float_concat ${latitude['from.ip']},${longitude['from.ip']}
|
220
|
+
float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
|
221
|
+
float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
|
222
|
+
string_concat ${latitude['from.ip']},${longitude['from.ip']}
|
223
|
+
string_array [${city['from.ip']}, ${country_name['from.ip']}]
|
224
|
+
string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
|
225
|
+
unknown_city ${city['unknown_key']}
|
226
|
+
undefined ${city['undefined']}
|
227
|
+
broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
|
228
|
+
broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
|
229
|
+
</record>
|
230
|
+
remove_tag_prefix input.
|
231
|
+
tag qqwry.${tag}
|
232
|
+
], 'input.access')
|
233
|
+
d1.run do
|
234
|
+
d1.emit({'from' => {'ip' => '66.102.3.80'}})
|
235
|
+
d1.emit({'message' => 'missing field'})
|
236
|
+
end
|
237
|
+
emits = d1.emits
|
238
|
+
assert_equal 2, emits.length
|
239
|
+
|
240
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
241
|
+
assert_equal 'Mountain View', emits[0][2]['from_city']
|
242
|
+
assert_equal 'United States', emits[0][2]['from_country']
|
243
|
+
assert_equal 37.4192008972168, emits[0][2]['latitude']
|
244
|
+
assert_equal -122.05740356445312, emits[0][2]['longitude']
|
245
|
+
assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['float_concat']
|
246
|
+
assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['float_array']
|
247
|
+
float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
|
248
|
+
assert_equal float_nest, emits[0][2]['float_nest']
|
249
|
+
assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['string_concat']
|
250
|
+
assert_equal ["Mountain View", "United States"], emits[0][2]['string_array']
|
251
|
+
string_nest = {"city" => "Mountain View", "country_name" => "United States"}
|
252
|
+
assert_equal string_nest, emits[0][2]['string_nest']
|
253
|
+
assert_equal nil, emits[0][2]['unknown_city']
|
254
|
+
assert_equal nil, emits[0][2]['undefined']
|
255
|
+
assert_equal [-122.05740356445312, nil], emits[0][2]['broken_array1']
|
256
|
+
assert_equal [nil, nil], emits[0][2]['broken_array2']
|
257
|
+
|
258
|
+
assert_equal nil, emits[1][2]['from_city']
|
259
|
+
assert_equal nil, emits[1][2]['from_country']
|
260
|
+
assert_equal nil, emits[1][2]['latitude']
|
261
|
+
assert_equal nil, emits[1][2]['longitude']
|
262
|
+
assert_equal ',', emits[1][2]['float_concat']
|
263
|
+
assert_equal [nil, nil], emits[1][2]['float_array']
|
264
|
+
float_nest = {"lat" => nil, "lon" => nil}
|
265
|
+
assert_equal float_nest, emits[1][2]['float_nest']
|
266
|
+
assert_equal ',', emits[1][2]['string_concat']
|
267
|
+
assert_equal [nil, nil], emits[1][2]['string_array']
|
268
|
+
string_nest = {"city" => nil, "country_name" => nil}
|
269
|
+
assert_equal string_nest, emits[1][2]['string_nest']
|
270
|
+
assert_equal nil, emits[1][2]['unknown_city']
|
271
|
+
assert_equal nil, emits[1][2]['undefined']
|
272
|
+
assert_equal [nil, nil], emits[1][2]['broken_array1']
|
273
|
+
assert_equal [nil, nil], emits[1][2]['broken_array2']
|
274
|
+
end
|
275
|
+
|
276
|
+
def test_emit_record_directive_multiple_record
|
277
|
+
d1 = create_driver(%[
|
278
|
+
qqwry_lookup_key from.ip, to.ip
|
279
|
+
<record>
|
280
|
+
from_city ${city['from.ip']}
|
281
|
+
to_city ${city['to.ip']}
|
282
|
+
from_country ${country_name['from.ip']}
|
283
|
+
to_country ${country_name['to.ip']}
|
284
|
+
string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
|
285
|
+
</record>
|
286
|
+
remove_tag_prefix input.
|
287
|
+
tag qqwry.${tag}
|
288
|
+
], 'input.access')
|
289
|
+
d1.run do
|
290
|
+
d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
|
291
|
+
d1.emit({'message' => 'missing field'})
|
292
|
+
end
|
293
|
+
emits = d1.emits
|
294
|
+
assert_equal 2, emits.length
|
295
|
+
|
296
|
+
assert_equal 'qqwry.access', emits[0][0] # tag
|
297
|
+
assert_equal 'Mountain View', emits[0][2]['from_city']
|
298
|
+
assert_equal 'United States', emits[0][2]['from_country']
|
299
|
+
assert_equal 'Musashino', emits[0][2]['to_city']
|
300
|
+
assert_equal 'Japan', emits[0][2]['to_country']
|
301
|
+
assert_equal ['United States','Japan'], emits[0][2]['string_array']
|
302
|
+
|
303
|
+
assert_equal nil, emits[1][2]['from_city']
|
304
|
+
assert_equal nil, emits[1][2]['to_city']
|
305
|
+
assert_equal nil, emits[1][2]['from_country']
|
306
|
+
assert_equal nil, emits[1][2]['to_country']
|
307
|
+
assert_equal [nil, nil], emits[1][2]['string_array']
|
308
|
+
end
|
309
|
+
end
|
metadata
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-qqwry
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Chris Song
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-05-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluent-mixin-rewrite-tag-name
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: qqwry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description:
|
84
|
+
email:
|
85
|
+
- fakechris@gmail.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- ".gitignore"
|
91
|
+
- Gemfile
|
92
|
+
- LICENSE
|
93
|
+
- README.md
|
94
|
+
- Rakefile
|
95
|
+
- data/qqwry.dat
|
96
|
+
- fluent-plugin-qqwry.gemspec
|
97
|
+
- lib/fluent/plugin/out_qqwry.rb
|
98
|
+
- test/helper.rb
|
99
|
+
- test/plugin/test_out_qqwry.rb
|
100
|
+
homepage: https://github.com/fakechris/fluent-plugin-qqwry
|
101
|
+
licenses:
|
102
|
+
- Apache License, Version 2.0
|
103
|
+
metadata: {}
|
104
|
+
post_install_message:
|
105
|
+
rdoc_options: []
|
106
|
+
require_paths:
|
107
|
+
- lib
|
108
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
109
|
+
requirements:
|
110
|
+
- - ">="
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '0'
|
113
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
requirements: []
|
119
|
+
rubyforge_project:
|
120
|
+
rubygems_version: 2.2.2
|
121
|
+
signing_key:
|
122
|
+
specification_version: 4
|
123
|
+
summary: Fluentd Output plugin to add information about geographical location of IP
|
124
|
+
addresses with QQWry databases.
|
125
|
+
test_files:
|
126
|
+
- test/helper.rb
|
127
|
+
- test/plugin/test_out_qqwry.rb
|