fluent-plugin-vertica-query 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.travis.yml +6 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +14 -0
- data/README.md +69 -0
- data/Rakefile +10 -0
- data/fluent-plugin-vertica-query.gemspec +20 -0
- data/lib/fluent/plugin/in_vertica_query.rb +100 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_in_vertica_query.rb +40 -0
- metadata +99 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: abc50c357a3a49197555d92cd9f4de03e4fa5b4a
|
4
|
+
data.tar.gz: 295737f289ead087497e5fb2c50c79f6695a008e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ca7dec83f73c36e5a0e21205295ba05d0b0a63465a27f82b6cf8455a10c65488efd19f398213037d68fd0b6eb07ad8f214d0284fe1258da835f8d802e11f87f6
|
7
|
+
data.tar.gz: 24dc3efbc6d94f4b5b1785d738e7dc2b17b7f35fe2424fc50d1afab513d03aee10bf8e0301e1127006063ffea9869fa9780ef52e86e1ecf514cf321a98add66d
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (c) 2012- Kentaro Yoshida
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/README.md
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
fluent-plugin-vertica-query
|
2
|
+
===========================
|
3
|
+
|
4
|
+
Fluentd Input plugin to execute vertica query and fetch rows. It is useful for stationary interval metrics measurement.
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
install with gem or fluent-gem command as:
|
9
|
+
|
10
|
+
```
|
11
|
+
# for fluentd
|
12
|
+
$ gem install fluent-plugin-vertica-query
|
13
|
+
|
14
|
+
# for td-agent
|
15
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-vertica-query
|
16
|
+
```
|
17
|
+
|
18
|
+
## Configuration
|
19
|
+
|
20
|
+
### Config Sample
|
21
|
+
`````
|
22
|
+
<source>
|
23
|
+
type vertica_query
|
24
|
+
host localhost # Optional (default: localhost)
|
25
|
+
port 5433 # Optional (default: 5433)
|
26
|
+
username nagios # Optional (default: dbadmin)
|
27
|
+
password passw0rd # Optional (default not set)
|
28
|
+
interval 30s # Optional (default: 1m)
|
29
|
+
database db # Optional (default: not set as there is only one database per cluster)
|
30
|
+
ssl true # Optional connection via ssl (default: false)
|
31
|
+
role pseudosuperuser # Optional additional role(s) to set for the user
|
32
|
+
search_path test_date # Optional (default: not set)
|
33
|
+
tag input.vertica # Required
|
34
|
+
query SELECT * FROM V_CATALOG.RESOURCE_POOLS; # Required
|
35
|
+
# record hostname into message.
|
36
|
+
record_host yes # Optional (default: no)
|
37
|
+
# multi row results into nested record or separated message.
|
38
|
+
nest_result yes # Optional (default: no)
|
39
|
+
nest_key data # Optional (default: result)
|
40
|
+
# record the number of lines of a query result
|
41
|
+
row_count yes # Optional (default: no)
|
42
|
+
row_count_key row_count # Optional (default: row_count)
|
43
|
+
</source>
|
44
|
+
|
45
|
+
<match input.vertica>
|
46
|
+
type stdout
|
47
|
+
</match>
|
48
|
+
`````
|
49
|
+
|
50
|
+
### Output Sample
|
51
|
+
record_hostname: yes, nest_result: no
|
52
|
+
`````
|
53
|
+
input.vertica: {"hostname":"myhost.example.com","Variable_name":"thread_cache_size","Value":"16"}
|
54
|
+
input.vertica: {"hostname":"myhost.example.com","Variable_name":"thread_stack","Value":"262144"}
|
55
|
+
`````
|
56
|
+
record_hostname: yes, nest_result: yes, nest_key: data
|
57
|
+
`````
|
58
|
+
input.vertica: {"hostname":"myhost.example.com","data":[{"Variable_name":"thread_cache_size","Value":"16"},{"Variable_name":"thread_stack","Value":"262144"}]}
|
59
|
+
`````
|
60
|
+
record_hostname: yes, nest_result: yes, nest_key: data, row_count: yes, row_count_key: row_count
|
61
|
+
`````
|
62
|
+
input.vertica: {"hostname":"myhost.example.com","row_count":2,"data":[{"Variable_name":"thread_cache_size","Value":"16"},{"Variable_name":"thread_stack","Value":"262144"}]}
|
63
|
+
`````
|
64
|
+
|
65
|
+
### Example Query
|
66
|
+
* SELECT MAX(id) AS max_foo_id FROM foo_table;
|
67
|
+
* SELECT * FROM nodes;
|
68
|
+
* SELECT * FROM V_CATALOG.RESOURCE_POOLS;
|
69
|
+
* SELECT user_id, memory_cap_kb FROM users WHERE resource_pool='general';
|
data/Rakefile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = "fluent-plugin-vertica-query"
|
6
|
+
s.version = "0.0.7"
|
7
|
+
s.authors = ["John Dodson"]
|
8
|
+
s.email = ["john.dodson@intentmedia.com"]
|
9
|
+
s.homepage = "https://github.com/intentmedia/fluent-plugin-vertica-query"
|
10
|
+
s.summary = %q{Fluentd Input plugin to execute Vertica query and fetch rows. It is useful for stationary interval metrics measurement. Forked from Kentaro Yoshida's fluent-plugin-mysql-query gem.}
|
11
|
+
|
12
|
+
s.files = `git ls-files`.split("\n")
|
13
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
14
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
15
|
+
s.require_paths = ["lib"]
|
16
|
+
|
17
|
+
s.add_development_dependency "rake"
|
18
|
+
s.add_runtime_dependency "fluentd"
|
19
|
+
s.add_runtime_dependency "vertica"
|
20
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
module Fluent
|
2
|
+
class VerticaQueryInput < Fluent::Input
|
3
|
+
Plugin.register_input('vertica_query', self)
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
require 'vertica'
|
7
|
+
require 'socket'
|
8
|
+
super
|
9
|
+
end
|
10
|
+
|
11
|
+
config_param :host, :string, :default => 'localhost'
|
12
|
+
config_param :port, :integer, :default => 5433
|
13
|
+
config_param :username, :string, :default => 'dbadmin'
|
14
|
+
config_param :password, :string, :default => nil
|
15
|
+
config_param :database, :string, :default => nil
|
16
|
+
config_param :ssl, :bool, :default => false # use SSL for the connection
|
17
|
+
config_param :role, :string, :default => nil # the (additional) role(s) to enable for the user.
|
18
|
+
config_param :search_path, :string, :default => nil
|
19
|
+
config_param :encoding, :string, :default => 'utf8'
|
20
|
+
config_param :interval, :time, :default => '1m'
|
21
|
+
config_param :nest_result, :bool, :default => false
|
22
|
+
config_param :nest_key, :string, :default => 'result'
|
23
|
+
config_param :row_count, :bool, :default => false
|
24
|
+
config_param :row_count_key, :string, :default => 'row_count'
|
25
|
+
config_param :record_host, :bool, :default => false
|
26
|
+
config_param :tag, :string
|
27
|
+
config_param :query, :string
|
28
|
+
|
29
|
+
def configure(conf)
|
30
|
+
super
|
31
|
+
@hostname = nil
|
32
|
+
$log.info "adding vertica_query job: [#{@query}] interval: #{@interval}sec"
|
33
|
+
end
|
34
|
+
|
35
|
+
def start
|
36
|
+
@thread = Thread.new(&method(:run))
|
37
|
+
end
|
38
|
+
|
39
|
+
def shutdown
|
40
|
+
Thread.kill(@thread)
|
41
|
+
end
|
42
|
+
|
43
|
+
def run
|
44
|
+
loop do
|
45
|
+
tag = "#{@tag}".gsub('__HOSTNAME__', @host).gsub('${hostname}', @host)
|
46
|
+
record = Hash.new
|
47
|
+
record.store('host', @host) if @record_host
|
48
|
+
result = get_exec_result
|
49
|
+
record.store(@row_count_key, result.size) if @row_count
|
50
|
+
if (@nest_result)
|
51
|
+
record.store(@nest_key, result)
|
52
|
+
Engine.emit(tag, Engine.now, record)
|
53
|
+
else
|
54
|
+
result.each do |data|
|
55
|
+
Engine.emit(tag, Engine.now, record.merge(data))
|
56
|
+
end
|
57
|
+
end
|
58
|
+
sleep @interval
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def get_connection
|
63
|
+
begin
|
64
|
+
return Vertica.connect({
|
65
|
+
:host => @host,
|
66
|
+
:port => @port,
|
67
|
+
:username => @username,
|
68
|
+
:password => @password,
|
69
|
+
:database => @database,
|
70
|
+
:encoding => @encoding,
|
71
|
+
:reconnect => true
|
72
|
+
})
|
73
|
+
rescue Exception => e
|
74
|
+
$log.warn "vertica_query: #{e}"
|
75
|
+
sleep @interval
|
76
|
+
retry
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def query(query)
|
81
|
+
@vertica ||= get_connection
|
82
|
+
begin
|
83
|
+
return @vertica.query(query)
|
84
|
+
rescue Exception => e
|
85
|
+
$log.warn "vertica_query: #{e}"
|
86
|
+
sleep @interval
|
87
|
+
retry
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def get_exec_result
|
92
|
+
result = Array.new
|
93
|
+
stmt = query(@query)
|
94
|
+
stmt.each do |row|
|
95
|
+
result.push(row)
|
96
|
+
end
|
97
|
+
return result
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/in_vertica_query'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class VerticaQueryInputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
host localhost
|
10
|
+
port 5433
|
11
|
+
interval 30
|
12
|
+
tag input.vertica
|
13
|
+
query \echo test
|
14
|
+
record_host yes
|
15
|
+
]
|
16
|
+
|
17
|
+
def create_driver(conf=CONFIG,tag='test')
|
18
|
+
Fluent::Test::OutputTestDriver.new(Fluent::VerticaQueryInput, tag).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_configure
|
22
|
+
assert_raise(Fluent::ConfigError) {
|
23
|
+
d = create_driver('')
|
24
|
+
}
|
25
|
+
d = create_driver %[
|
26
|
+
host localhost
|
27
|
+
port 5433
|
28
|
+
interval 30
|
29
|
+
tag input.vertica
|
30
|
+
query \echo test
|
31
|
+
record_host yes
|
32
|
+
]
|
33
|
+
assert_equal 'localhost', d.instance.host
|
34
|
+
assert_equal 5433, d.instance.port
|
35
|
+
assert_equal 30, d.instance.interval
|
36
|
+
assert_equal 'input.vertica', d.instance.tag
|
37
|
+
assert_equal true, d.instance.record_host
|
38
|
+
assert_equal false, d.instance.nest_result
|
39
|
+
end
|
40
|
+
end
|
metadata
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-vertica-query
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.7
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- John Dodson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-02-09 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: vertica
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
- john.dodson@intentmedia.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
64
|
+
- Gemfile
|
65
|
+
- LICENSE.txt
|
66
|
+
- README.md
|
67
|
+
- Rakefile
|
68
|
+
- fluent-plugin-vertica-query.gemspec
|
69
|
+
- lib/fluent/plugin/in_vertica_query.rb
|
70
|
+
- test/helper.rb
|
71
|
+
- test/plugin/test_in_vertica_query.rb
|
72
|
+
homepage: https://github.com/intentmedia/fluent-plugin-vertica-query
|
73
|
+
licenses: []
|
74
|
+
metadata: {}
|
75
|
+
post_install_message:
|
76
|
+
rdoc_options: []
|
77
|
+
require_paths:
|
78
|
+
- lib
|
79
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
requirements: []
|
90
|
+
rubyforge_project:
|
91
|
+
rubygems_version: 2.2.2
|
92
|
+
signing_key:
|
93
|
+
specification_version: 4
|
94
|
+
summary: Fluentd Input plugin to execute Vertica query and fetch rows. It is useful
|
95
|
+
for stationary interval metrics measurement. Forked from Kentaro Yoshida's fluent-plugin-mysql-query
|
96
|
+
gem.
|
97
|
+
test_files:
|
98
|
+
- test/helper.rb
|
99
|
+
- test/plugin/test_in_vertica_query.rb
|