fluent-plugin-qqwry 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 13b7b4e4a05362dd611e2ca9bd8d9cf1fb8fa5b5
4
+ data.tar.gz: 811fcc85eaa7dae22545904f30f4687a2c11a152
5
+ SHA512:
6
+ metadata.gz: 054d4542cb901f72524851855e6229cd52311580ab4cd5a9b75e88e1448ab4b11989f086239d6582061ef787d096179b0bce034e9b88f796d69dc1dea6de46e7
7
+ data.tar.gz: 7ca2d977db399a85c87c1531a74d62a4fe23e42e6f49c67cc046c63f6bacf5dbcca58ece36d834db44954c0186967034cf2df13a70ed3ccf6c7bdd7e2c5ba61c
data/.gitignore ADDED
@@ -0,0 +1,21 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ coverage
6
+ InstalledFiles
7
+ lib/bundler/man
8
+ pkg
9
+ rdoc
10
+ spec/reports
11
+ test/tmp
12
+ test/version_tmp
13
+ tmp
14
+ *.swp
15
+ vendor/
16
+ Gemfile.lock
17
+
18
+ # YARD artifacts
19
+ .yardoc
20
+ _yardoc
21
+ doc/
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-geoip.gemspec
4
+ gemspec
5
+
6
+ gem "qqwry"
data/LICENSE ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (c) 2013- Kentaro Yoshida
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
14
+
data/README.md ADDED
@@ -0,0 +1,150 @@
1
+ # fluent-plugin-qqwry
2
+
3
+ Fluentd Output plugin to add information about geographical location of IP addresses with QQWry databases.
4
+
5
+ fluent-plugin-geoip has bundled qqwry.dat
6
+
7
+ ## Dependency
8
+
9
+ before use, install dependent library as:
10
+
11
+ ## Installation
12
+
13
+ install with `gem` or `fluent-gem` command as:
14
+
15
+ ```bash
16
+ # for fluentd
17
+ $ gem install fluent-plugin-qqwry
18
+
19
+ # for td-agent
20
+ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-qqwry
21
+ ```
22
+
23
+ ## Usage
24
+
25
+ ```xml
26
+ <match access.apache>
27
+ type qqwry
28
+
29
+ # Specify one or more qqwry lookup field which has ip address (default: host)
30
+ # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
31
+ qqwry_lookup_key host
32
+
33
+ # Specify optional qqwry database (using bundled GeoLiteCity databse by default)
34
+ qqwry_database '/path/to/your/qqwry.dat'
35
+
36
+ # Set adding field with placeholder (more than one settings are required.)
37
+ <record>
38
+ city ${city['host']}
39
+ area ${area_code['host']}
40
+ country ${country['host']}
41
+ province ${province['host']}
42
+ </record>
43
+
44
+ # Settings for tag
45
+ remove_tag_prefix access.
46
+ tag qqwry.${tag}
47
+
48
+ # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
49
+ log_level info
50
+
51
+ # Set buffering time (default: 0s)
52
+ flush_interval 1s
53
+ </match>
54
+ ```
55
+
56
+ #### Tips: how to geolocate multiple key
57
+
58
+ ```xml
59
+ <match access.apache>
60
+ type qqwry
61
+ qqwry_lookup_key user1_host, user2_host
62
+ <record>
63
+ user1_city ${city['user1_host']}
64
+ user2_city ${city['user2_host']}
65
+ </record>
66
+ remove_tag_prefix access.
67
+ tag qqwry.${tag}
68
+ </match>
69
+ ```
70
+
71
+ ## Tutorial
72
+
73
+ #### configuration
74
+
75
+ ```xml
76
+ <source>
77
+ type forward
78
+ </source>
79
+
80
+ <match test.qqwry>
81
+ type copy
82
+ <store>
83
+ type stdout
84
+ </store>
85
+ <store>
86
+ type qqwry
87
+ qqwry_lookup_key host
88
+ <record>
89
+ city ${city['host']}
90
+ lat ${latitude['host']}
91
+ lon ${longitude['host']}
92
+ </record>
93
+ remove_tag_prefix test.
94
+ tag debug.${tag}
95
+ </store>
96
+ </match>
97
+
98
+ <match debug.**>
99
+ type stdout
100
+ </match>
101
+ ```
102
+
103
+ #### result
104
+
105
+ ```bash
106
+ # forward record with Google's ip address.
107
+ $ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.qqwry
108
+
109
+ # check the result at stdout
110
+ $ tail /var/log/td-agent/td-agent.log
111
+ 2013-08-04 16:21:32 +0900 test.qqwry: {"host":"66.102.9.80","message":"test"}
112
+ 2013-08-04 16:21:32 +0900 debug.qqwry: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
113
+ ```
114
+
115
+ ## Parameters
116
+
117
+ * `include_tag_key` (default: false)
118
+ * `tag_key`
119
+
120
+ Add original tag name into filtered record using SetTagKeyMixin.<br />
121
+ Further details are written at http://docs.fluentd.org/articles/in_exec
122
+
123
+ * `remove_tag_prefix`
124
+ * `remove_tag_suffix`
125
+ * `add_tag_prefix`
126
+ * `add_tag_suffix`
127
+
128
+ Set one or more option are required unless using `tag` option for editing tag name. (HandleTagNameMixin feature)
129
+
130
+ * `tag`
131
+
132
+ On using this option with tag placeholder like `tag qqwry.${tag}` (test code is available at [test_out_qqwry.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
133
+
134
+ * `flush_interval` (default: 0 sec)
135
+
136
+ ## Contributing
137
+
138
+ 1. Fork it
139
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
140
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
141
+ 4. Push to the branch (`git push origin my-new-feature`)
142
+ 5. Create new Pull Request
143
+
144
+ ## Copyright
145
+
146
+ Copyright (c) 2014- Chris Song ([@fakechris](http://weibo.com/songchris))
147
+
148
+ ## License
149
+
150
+ Apache License, Version 2.0
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+ Rake::TestTask.new(:test) do |test|
4
+ test.libs << 'lib' << 'test'
5
+ test.pattern = 'test/**/test_*.rb'
6
+ test.verbose = true
7
+ end
8
+
9
+ task :default => :test
data/data/qqwry.dat ADDED
Binary file
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-plugin-qqwry"
7
+ spec.version = "0.0.1"
8
+ spec.authors = ["Chris Song"]
9
+ spec.email = ["fakechris@gmail.com"]
10
+ spec.summary = %q{Fluentd Output plugin to add information about geographical location of IP addresses with QQWry databases.}
11
+ spec.homepage = "https://github.com/fakechris/fluent-plugin-qqwry"
12
+ spec.license = "Apache License, Version 2.0"
13
+
14
+ spec.files = `git ls-files`.split($/)
15
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
16
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
17
+ spec.require_paths = ["lib"]
18
+
19
+ spec.add_development_dependency "bundler"
20
+ spec.add_development_dependency "rake"
21
+ spec.add_runtime_dependency "fluentd"
22
+ spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
23
+ spec.add_runtime_dependency "qqwry"
24
+ end
@@ -0,0 +1,166 @@
1
+ require 'fluent/mixin/rewrite_tag_name'
2
+
3
+ class Fluent::QQWryOutput < Fluent::BufferedOutput
4
+ Fluent::Plugin.register_output('qqwry', self)
5
+
6
+ REGEXP_JSON = /(^[\[\{].+[\]\}]$|^[\d\.\-]+$)/
7
+ REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<qqwry_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
8
+ REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
9
+ QQWRY_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
10
+
11
+ config_param :qqwry_database, :string, :default => File.dirname(__FILE__) + '/../../../data/qqwry.dat'
12
+ config_param :qqwry_lookup_key, :string, :default => 'host'
13
+ config_param :tag, :string, :default => nil
14
+
15
+ include Fluent::HandleTagNameMixin
16
+ include Fluent::SetTagKeyMixin
17
+ config_set_default :include_tag_key, false
18
+
19
+ include Fluent::Mixin::RewriteTagName
20
+ config_param :hostname_command, :string, :default => 'hostname'
21
+
22
+ config_param :flush_interval, :time, :default => 0
23
+ config_param :log_level, :string, :default => 'warn'
24
+
25
+ # Define `log` method for v0.10.42 or earlier
26
+ unless method_defined?(:log)
27
+ define_method("log") { $log }
28
+ end
29
+
30
+ def initialize
31
+ require 'qqwry'
32
+
33
+ super
34
+ end
35
+
36
+ def configure(conf)
37
+ super
38
+
39
+ @map = {}
40
+ @qqwry_lookup_key = @qqwry_lookup_key.split(/\s*,\s*/)
41
+
42
+ # enable_key_* format (legacy format)
43
+ conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
44
+ qqwry_key = key.sub('enable_key_','')
45
+ raise Fluent::ConfigError, "qqwry: unsupported key #{qqwry_key}" unless QQWRY_KEYS.include?(qqwry_key)
46
+ @qqwry_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
47
+ if record_key.nil?
48
+ raise Fluent::ConfigError, "qqwry: missing value found at '#{key} #{lookup_field}'"
49
+ end
50
+ @map.store(record_key, "${#{qqwry_key}['#{lookup_field}']}")
51
+ end
52
+ end
53
+ if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
54
+ log.warn "qqwry: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
55
+ log.warn "qqwry: for further details referable to https://github.com/fakechris/fluent-plugin-qqwry"
56
+ end
57
+
58
+ # <record></record> directive
59
+ conf.elements.select { |element| element.name == 'record' }.each { |element|
60
+ element.each_pair { |k, v|
61
+ element.has_key?(k) # to suppress unread configuration warning
62
+ @map[k] = v
63
+ validate_json = Proc.new {
64
+ begin
65
+ dummy_text = Yajl::Encoder.encode('dummy_text')
66
+ Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
67
+ rescue Yajl::ParseError => e
68
+ raise Fluent::ConfigError, "qqwry: failed to parse '#{v}' as json."
69
+ end
70
+ }
71
+ validate_json.call if v.match(REGEXP_JSON)
72
+ }
73
+ }
74
+ @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
75
+ @placeholder_keys.each do |key|
76
+ qqwry_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:qqwry_key]
77
+ raise Fluent::ConfigError, "qqwry: unsupported key #{qqwry_key}" unless QQWRY_KEYS.include?(qqwry_key)
78
+ end
79
+ @placeholder_expander = PlaceholderExpander.new
80
+
81
+ if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
82
+ raise Fluent::ConfigError, "qqwry: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
83
+ end
84
+
85
+ @qqwry = QQWry::Database.new(@qqwry_database)
86
+ end
87
+
88
+ def start
89
+ super
90
+ end
91
+
92
+ def format(tag, time, record)
93
+ [tag, time, record].to_msgpack
94
+ end
95
+
96
+ def shutdown
97
+ super
98
+ end
99
+
100
+ def write(chunk)
101
+ chunk.msgpack_each do |tag, time, record|
102
+ Fluent::Engine.emit(tag, time, add_qqwry_field(record))
103
+ end
104
+ end
105
+
106
+ private
107
+ def add_qqwry_field(record)
108
+ placeholder = create_placeholder(geolocate(get_address(record)))
109
+ @map.each do |record_key, value|
110
+ if value.match(REGEXP_PLACEHOLDER_SINGLE)
111
+ rewrited = placeholder[value]
112
+ elsif value.match(REGEXP_JSON)
113
+ rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
114
+ Yajl::Encoder.encode(placeholder[match])
115
+ }
116
+ rewrited = parse_json(rewrited)
117
+ else
118
+ rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
119
+ end
120
+ record.store(record_key, rewrited)
121
+ end
122
+ return record
123
+ end
124
+
125
+ def parse_json(message)
126
+ begin
127
+ return Yajl::Parser.parse(message)
128
+ rescue Yajl::ParseError => e
129
+ log.info "qqwry: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
130
+ return nil
131
+ end
132
+ end
133
+
134
+ def get_address(record)
135
+ address = {}
136
+ @qqwry_lookup_key.each do |field|
137
+ key = field.split('.')
138
+ obj = record
139
+ key.each {|k|
140
+ break obj = nil if not obj.has_key?(k)
141
+ obj = obj[k]
142
+ }
143
+ address.store(field, obj)
144
+ end
145
+ return address
146
+ end
147
+
148
+ def geolocate(addresses)
149
+ geodata = {}
150
+ addresses.each do |field, ip|
151
+ geo = ip.nil? ? nil : @qqwry.query(ip)
152
+ geodata.store(field, geo)
153
+ end
154
+ return geodata
155
+ end
156
+
157
+ def create_placeholder(geodata)
158
+ placeholder = {}
159
+ @placeholder_keys.each do |placeholder_key|
160
+ position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
161
+ next if position.nil? or geodata[position[:record_key]].nil?
162
+ placeholder.store(placeholder_key, geodata[position[:record_key]][position[:qqwry_key].to_sym])
163
+ end
164
+ return placeholder
165
+ end
166
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_geoip'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,309 @@
1
+ require 'helper'
2
+
3
+ class QQWryOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ qqwry_lookup_key host
10
+ enable_key_city qqwry_city
11
+ remove_tag_prefix input.
12
+ tag qqwry.${tag}
13
+ ]
14
+
15
+ def create_driver(conf=CONFIG,tag='test')
16
+ Fluent::Test::OutputTestDriver.new(Fluent::QQWryOutput, tag).configure(conf)
17
+ end
18
+
19
+ def test_configure
20
+ assert_raise(Fluent::ConfigError) {
21
+ d = create_driver('')
22
+ }
23
+ assert_raise(Fluent::ConfigError) {
24
+ d = create_driver('enable_key_cities')
25
+ }
26
+ d = create_driver %[
27
+ enable_key_city qqwry_city
28
+ remove_tag_prefix input.
29
+ tag qqwry.${tag}
30
+ ]
31
+ assert_equal 'qqwry_city', d.instance.config['enable_key_city']
32
+
33
+ # multiple key config
34
+ d = create_driver %[
35
+ qqwry_lookup_key from.ip, to.ip
36
+ enable_key_city from_city, to_city
37
+ remove_tag_prefix input.
38
+ tag qqwry.${tag}
39
+ ]
40
+ assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
41
+
42
+ # multiple key config (bad configure)
43
+ assert_raise(Fluent::ConfigError) {
44
+ d = create_driver %[
45
+ qqwry_lookup_key from.ip, to.ip
46
+ enable_key_city from_city
47
+ enable_key_region from_region
48
+ remove_tag_prefix input.
49
+ tag qqwry.${tag}
50
+ ]
51
+ }
52
+
53
+ # invalid json structure
54
+ assert_raise(Fluent::ConfigError) {
55
+ d = create_driver %[
56
+ qqwry_lookup_key host
57
+ <record>
58
+ invalid_json {"foo" => 123}
59
+ </record>
60
+ remove_tag_prefix input.
61
+ tag qqwry.${tag}
62
+ ]
63
+ }
64
+ assert_raise(Fluent::ConfigError) {
65
+ d = create_driver %[
66
+ qqwry_lookup_key host
67
+ <record>
68
+ invalid_json {"foo" : string, "bar" : 123}
69
+ </record>
70
+ remove_tag_prefix input.
71
+ tag qqwry.${tag}
72
+ ]
73
+ }
74
+ end
75
+
76
+ def test_emit
77
+ d1 = create_driver(CONFIG, 'input.access')
78
+ d1.run do
79
+ d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
80
+ d1.emit({'message' => 'missing field'})
81
+ end
82
+ emits = d1.emits
83
+ assert_equal 2, emits.length
84
+ assert_equal 'qqwry.access', emits[0][0] # tag
85
+ assert_equal 'Mountain View', emits[0][2]['qqwry_city']
86
+ assert_equal nil, emits[1][2]['qqwry_city']
87
+ end
88
+
89
+ def test_emit_tag_option
90
+ d1 = create_driver(%[
91
+ qqwry_lookup_key host
92
+ <record>
93
+ qqwry_city ${city['host']}
94
+ </record>
95
+ remove_tag_prefix input.
96
+ tag qqwry.${tag}
97
+ ], 'input.access')
98
+ d1.run do
99
+ d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
100
+ d1.emit({'message' => 'missing field'})
101
+ end
102
+ emits = d1.emits
103
+ assert_equal 2, emits.length
104
+ assert_equal 'qqwry.access', emits[0][0] # tag
105
+ assert_equal 'Mountain View', emits[0][2]['qqwry_city']
106
+ assert_equal nil, emits[1][2]['qqwry_city']
107
+ end
108
+
109
+ def test_emit_tag_parts
110
+ d1 = create_driver(%[
111
+ qqwry_lookup_key host
112
+ <record>
113
+ qqwry_city ${city['host']}
114
+ </record>
115
+ tag qqwry.${tag_parts[1]}.${tag_parts[2..3]}.${tag_parts[-1]}
116
+ ], '0.1.2.3')
117
+ d1.run do
118
+ d1.emit({'host' => '66.102.3.80'})
119
+ end
120
+ emits = d1.emits
121
+ assert_equal 1, emits.length
122
+ assert_equal 'qqwry.1.2.3.3', emits[0][0] # tag
123
+ assert_equal 'Mountain View', emits[0][2]['qqwry_city']
124
+ end
125
+
126
+ def test_emit_nested_attr
127
+ d1 = create_driver(%[
128
+ qqwry_lookup_key host.ip
129
+ enable_key_city qqwry_city
130
+ remove_tag_prefix input.
131
+ add_tag_prefix qqwry.
132
+ ], 'input.access')
133
+ d1.run do
134
+ d1.emit({'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'})
135
+ d1.emit({'message' => 'missing field'})
136
+ end
137
+ emits = d1.emits
138
+ assert_equal 2, emits.length
139
+ assert_equal 'qqwry.access', emits[0][0] # tag
140
+ assert_equal 'Mountain View', emits[0][2]['qqwry_city']
141
+ assert_equal nil, emits[1][2]['qqwry_city']
142
+ end
143
+
144
+ def test_emit_with_unknown_address
145
+ d1 = create_driver(CONFIG, 'input.access')
146
+ d1.run do
147
+ # 203.0.113.1 is a test address described in RFC5737
148
+ d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
149
+ d1.emit({'host' => '0', 'message' => 'invalid ip'})
150
+ end
151
+ emits = d1.emits
152
+ assert_equal 2, emits.length
153
+ assert_equal 'qqwry.access', emits[0][0] # tag
154
+ assert_equal nil, emits[0][2]['qqwry_city']
155
+ assert_equal 'qqwry.access', emits[1][0] # tag
156
+ assert_equal nil, emits[1][2]['qqwry_city']
157
+ end
158
+
159
+ def test_emit_multiple_key
160
+ d1 = create_driver(%[
161
+ qqwry_lookup_key from.ip, to.ip
162
+ enable_key_city from_city, to_city
163
+ remove_tag_prefix input.
164
+ add_tag_prefix qqwry.
165
+ ], 'input.access')
166
+ d1.run do
167
+ d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
168
+ d1.emit({'message' => 'missing field'})
169
+ end
170
+ emits = d1.emits
171
+ assert_equal 2, emits.length
172
+ assert_equal 'qqwry.access', emits[0][0] # tag
173
+ assert_equal 'Mountain View', emits[0][2]['from_city']
174
+ assert_equal 'Musashino', emits[0][2]['to_city']
175
+ assert_equal nil, emits[1][2]['from_city']
176
+ assert_equal nil, emits[1][2]['to_city']
177
+ end
178
+
179
+ def test_emit_multiple_key_multiple_record
180
+ d1 = create_driver(%[
181
+ qqwry_lookup_key from.ip, to.ip
182
+ enable_key_city from_city, to_city
183
+ enable_key_country_name from_country, to_country
184
+ remove_tag_prefix input.
185
+ add_tag_prefix qqwry.
186
+ ], 'input.access')
187
+ d1.run do
188
+ d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
189
+ d1.emit({'from' => {'ip' => '66.102.3.80'}})
190
+ d1.emit({'message' => 'missing field'})
191
+ end
192
+ emits = d1.emits
193
+ assert_equal 3, emits.length
194
+ assert_equal 'qqwry.access', emits[0][0] # tag
195
+ assert_equal 'Mountain View', emits[0][2]['from_city']
196
+ assert_equal 'United States', emits[0][2]['from_country']
197
+ assert_equal 'Musashino', emits[0][2]['to_city']
198
+ assert_equal 'Japan', emits[0][2]['to_country']
199
+
200
+ assert_equal 'Mountain View', emits[1][2]['from_city']
201
+ assert_equal 'United States', emits[1][2]['from_country']
202
+ assert_equal nil, emits[1][2]['to_city']
203
+ assert_equal nil, emits[1][2]['to_country']
204
+
205
+ assert_equal nil, emits[2][2]['from_city']
206
+ assert_equal nil, emits[2][2]['from_country']
207
+ assert_equal nil, emits[2][2]['to_city']
208
+ assert_equal nil, emits[2][2]['to_country']
209
+ end
210
+
211
+ def test_emit_record_directive
212
+ d1 = create_driver(%[
213
+ qqwry_lookup_key from.ip
214
+ <record>
215
+ from_city ${city['from.ip']}
216
+ from_country ${country_name['from.ip']}
217
+ latitude ${latitude['from.ip']}
218
+ longitude ${longitude['from.ip']}
219
+ float_concat ${latitude['from.ip']},${longitude['from.ip']}
220
+ float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
221
+ float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
222
+ string_concat ${latitude['from.ip']},${longitude['from.ip']}
223
+ string_array [${city['from.ip']}, ${country_name['from.ip']}]
224
+ string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
225
+ unknown_city ${city['unknown_key']}
226
+ undefined ${city['undefined']}
227
+ broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
228
+ broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
229
+ </record>
230
+ remove_tag_prefix input.
231
+ tag qqwry.${tag}
232
+ ], 'input.access')
233
+ d1.run do
234
+ d1.emit({'from' => {'ip' => '66.102.3.80'}})
235
+ d1.emit({'message' => 'missing field'})
236
+ end
237
+ emits = d1.emits
238
+ assert_equal 2, emits.length
239
+
240
+ assert_equal 'qqwry.access', emits[0][0] # tag
241
+ assert_equal 'Mountain View', emits[0][2]['from_city']
242
+ assert_equal 'United States', emits[0][2]['from_country']
243
+ assert_equal 37.4192008972168, emits[0][2]['latitude']
244
+ assert_equal -122.05740356445312, emits[0][2]['longitude']
245
+ assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['float_concat']
246
+ assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['float_array']
247
+ float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
248
+ assert_equal float_nest, emits[0][2]['float_nest']
249
+ assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['string_concat']
250
+ assert_equal ["Mountain View", "United States"], emits[0][2]['string_array']
251
+ string_nest = {"city" => "Mountain View", "country_name" => "United States"}
252
+ assert_equal string_nest, emits[0][2]['string_nest']
253
+ assert_equal nil, emits[0][2]['unknown_city']
254
+ assert_equal nil, emits[0][2]['undefined']
255
+ assert_equal [-122.05740356445312, nil], emits[0][2]['broken_array1']
256
+ assert_equal [nil, nil], emits[0][2]['broken_array2']
257
+
258
+ assert_equal nil, emits[1][2]['from_city']
259
+ assert_equal nil, emits[1][2]['from_country']
260
+ assert_equal nil, emits[1][2]['latitude']
261
+ assert_equal nil, emits[1][2]['longitude']
262
+ assert_equal ',', emits[1][2]['float_concat']
263
+ assert_equal [nil, nil], emits[1][2]['float_array']
264
+ float_nest = {"lat" => nil, "lon" => nil}
265
+ assert_equal float_nest, emits[1][2]['float_nest']
266
+ assert_equal ',', emits[1][2]['string_concat']
267
+ assert_equal [nil, nil], emits[1][2]['string_array']
268
+ string_nest = {"city" => nil, "country_name" => nil}
269
+ assert_equal string_nest, emits[1][2]['string_nest']
270
+ assert_equal nil, emits[1][2]['unknown_city']
271
+ assert_equal nil, emits[1][2]['undefined']
272
+ assert_equal [nil, nil], emits[1][2]['broken_array1']
273
+ assert_equal [nil, nil], emits[1][2]['broken_array2']
274
+ end
275
+
276
+ def test_emit_record_directive_multiple_record
277
+ d1 = create_driver(%[
278
+ qqwry_lookup_key from.ip, to.ip
279
+ <record>
280
+ from_city ${city['from.ip']}
281
+ to_city ${city['to.ip']}
282
+ from_country ${country_name['from.ip']}
283
+ to_country ${country_name['to.ip']}
284
+ string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
285
+ </record>
286
+ remove_tag_prefix input.
287
+ tag qqwry.${tag}
288
+ ], 'input.access')
289
+ d1.run do
290
+ d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
291
+ d1.emit({'message' => 'missing field'})
292
+ end
293
+ emits = d1.emits
294
+ assert_equal 2, emits.length
295
+
296
+ assert_equal 'qqwry.access', emits[0][0] # tag
297
+ assert_equal 'Mountain View', emits[0][2]['from_city']
298
+ assert_equal 'United States', emits[0][2]['from_country']
299
+ assert_equal 'Musashino', emits[0][2]['to_city']
300
+ assert_equal 'Japan', emits[0][2]['to_country']
301
+ assert_equal ['United States','Japan'], emits[0][2]['string_array']
302
+
303
+ assert_equal nil, emits[1][2]['from_city']
304
+ assert_equal nil, emits[1][2]['to_city']
305
+ assert_equal nil, emits[1][2]['from_country']
306
+ assert_equal nil, emits[1][2]['to_country']
307
+ assert_equal [nil, nil], emits[1][2]['string_array']
308
+ end
309
+ end
metadata ADDED
@@ -0,0 +1,127 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-qqwry
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Chris Song
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-05-29 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fluentd
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: fluent-mixin-rewrite-tag-name
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: qqwry
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description:
84
+ email:
85
+ - fakechris@gmail.com
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - ".gitignore"
91
+ - Gemfile
92
+ - LICENSE
93
+ - README.md
94
+ - Rakefile
95
+ - data/qqwry.dat
96
+ - fluent-plugin-qqwry.gemspec
97
+ - lib/fluent/plugin/out_qqwry.rb
98
+ - test/helper.rb
99
+ - test/plugin/test_out_qqwry.rb
100
+ homepage: https://github.com/fakechris/fluent-plugin-qqwry
101
+ licenses:
102
+ - Apache License, Version 2.0
103
+ metadata: {}
104
+ post_install_message:
105
+ rdoc_options: []
106
+ require_paths:
107
+ - lib
108
+ required_ruby_version: !ruby/object:Gem::Requirement
109
+ requirements:
110
+ - - ">="
111
+ - !ruby/object:Gem::Version
112
+ version: '0'
113
+ required_rubygems_version: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ requirements: []
119
+ rubyforge_project:
120
+ rubygems_version: 2.2.2
121
+ signing_key:
122
+ specification_version: 4
123
+ summary: Fluentd Output plugin to add information about geographical location of IP
124
+ addresses with QQWry databases.
125
+ test_files:
126
+ - test/helper.rb
127
+ - test/plugin/test_out_qqwry.rb