fluent-plugin-cratedb 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +57 -0
- data/Rakefile +10 -0
- data/VERSION +1 -0
- data/fluent-plugin-cratedb.gemspec +27 -0
- data/lib/fluent/plugin/out_cratedb.rb +114 -0
- metadata +95 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 827ea567aeac08e5e0f453e1fa72b5fd0b168f69
|
4
|
+
data.tar.gz: 2dc49370a61a21f3d22a353b0112472928ffd0dc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 85ee563ce046878641a8687b231d5b926d63be76da312ee78e1b1c7284799759f84f5dc0cf055830bdeb9cb728d30863efac6430e43ef264f1fbe8955afc406a
|
7
|
+
data.tar.gz: 252fe5d3c55e43714472d448c4265dead1dbac9c510316ba0152c241fde205d06aea13648a70835dddb022759937b74600b0f45aac5d2ed04832236905929bfb
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2016 - BUOM
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
# Fluent::Plugin::CrateDB, a plugin for [Fluentd](http://fluentd.org)
|
2
|
+
|
3
|
+
fluent-plugin-cratedb is a buffered output plugin for fluentd and CrateDB.
|
4
|
+
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
$ fluent-gem install fluent-plugin-cratedb
|
9
|
+
|
10
|
+
## Usage
|
11
|
+
|
12
|
+
`type` `createdb`
|
13
|
+
|
14
|
+
--------------
|
15
|
+
|
16
|
+
**Options:**
|
17
|
+
|
18
|
+
`host`: cratedb host (default: 127.0.0.1)
|
19
|
+
|
20
|
+
`port`: cratedb port (default: 4200)
|
21
|
+
|
22
|
+
`hosts`: an array of servers including ports [127.0.0.1:4200, 10.0.0.1:4201]
|
23
|
+
|
24
|
+
`table`: bulk insert table (require)
|
25
|
+
|
26
|
+
`column_names`: bulk insert column (require)
|
27
|
+
|
28
|
+
`key_names`: key name values (default `column_names`)
|
29
|
+
|
30
|
+
|
31
|
+
## Configuration Example
|
32
|
+
|
33
|
+
```
|
34
|
+
<match mylog.*>
|
35
|
+
@type cratedb
|
36
|
+
host localhost
|
37
|
+
port 4200
|
38
|
+
column_names id,user_name,created_at,updated_at
|
39
|
+
table users
|
40
|
+
flush_interval 10s
|
41
|
+
</match>
|
42
|
+
```
|
43
|
+
|
44
|
+
## Contributing
|
45
|
+
|
46
|
+
|
47
|
+
1. Fork it
|
48
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
49
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
50
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
51
|
+
5. Create new Pull Request
|
52
|
+
|
53
|
+
|
54
|
+
## Licence
|
55
|
+
|
56
|
+
|
57
|
+
This package was distributed under Apache-2.0 licence, see LICENCE file for details.
|
data/Rakefile
ADDED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
#require 'fluent/plugin/cratedb/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "fluent-plugin-cratedb"
|
8
|
+
spec.version = File.read('VERSION').strip
|
9
|
+
spec.authors = ["buom"]
|
10
|
+
spec.email = ["me@buom.io"]
|
11
|
+
|
12
|
+
spec.summary = %q{A plugin for Fluentd}
|
13
|
+
spec.description = %q{Fluent Output Plugin for CrateDB (http://crate.io)}
|
14
|
+
spec.homepage = "https://github.com/buom/fluent-plugin-cratedb"
|
15
|
+
spec.license = "Apache-2.0"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|bin)/}) }
|
18
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_dependency "crate_ruby", "~> 0.0.8"
|
23
|
+
spec.add_runtime_dependency "fluentd", "~> 0.12.0"
|
24
|
+
|
25
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
26
|
+
#spec.add_development_dependency "minitest"
|
27
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'crate_ruby'
|
2
|
+
require 'digest'
|
3
|
+
require 'fluent/output'
|
4
|
+
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
class Fluent::CratedbOutput < Fluent::BufferedOutput
|
8
|
+
Fluent::Plugin.register_output('cratedb', self)
|
9
|
+
|
10
|
+
include Fluent::SetTimeKeyMixin
|
11
|
+
|
12
|
+
config_param :host, :string, :default => 'localhost', :desc => "CrateDB host."
|
13
|
+
config_param :port, :integer, :default => 4200, :desc => "CrateDB port."
|
14
|
+
config_param :hosts, :array, :default => nil
|
15
|
+
config_param :http_options, :hash, :default => {}
|
16
|
+
config_param :column_names, :string, :desc => "Bulk insert column."
|
17
|
+
config_param :key_names, :string, :default => nil
|
18
|
+
config_param :table, :string, :desc => "Bulk insert table."
|
19
|
+
|
20
|
+
# Define `log` method for v0.10.42 or earlier
|
21
|
+
unless method_defined?(:log)
|
22
|
+
define_method("log") { $log }
|
23
|
+
end
|
24
|
+
|
25
|
+
def configure(conf)
|
26
|
+
super
|
27
|
+
|
28
|
+
if @column_names.nil?
|
29
|
+
fail Fluent::ConfigError, 'column_names MUST specified, but missing'
|
30
|
+
end
|
31
|
+
|
32
|
+
@column_names = @column_names.split(',').collect(&:strip)
|
33
|
+
@key_names = @key_names.nil? ? @column_names : @key_names.split(',').collect(&:strip)
|
34
|
+
|
35
|
+
@servers = @hosts ? @hosts : ["#{@host}:#{@port}"]
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
super
|
40
|
+
require 'logger'
|
41
|
+
opts = {:http_options => @http_options, :logger => Logger.new(STDERR)}
|
42
|
+
@client = CrateClient.new(@servers, opts)
|
43
|
+
end
|
44
|
+
|
45
|
+
def shutdown
|
46
|
+
super
|
47
|
+
end
|
48
|
+
|
49
|
+
def format(tag, time, record)
|
50
|
+
[tag, time, format_proc.call(tag, time, record)].to_msgpack
|
51
|
+
end
|
52
|
+
|
53
|
+
def write(chunk)
|
54
|
+
values = []
|
55
|
+
chunk.msgpack_each do |tag, time, data|
|
56
|
+
#data = format_proc.call(tag, time, data)
|
57
|
+
values << data
|
58
|
+
end
|
59
|
+
sql = "INSERT INTO #{@table} (#{@column_names.join(',')}) VALUES (#{ @column_names.map { |key| '?' }.join(',') })"
|
60
|
+
#log.info(sql)
|
61
|
+
@client.execute(sql, nil, values, @http_options)
|
62
|
+
|
63
|
+
#digest = Digest::SHA1.hexdigest(data)
|
64
|
+
#@client.blob_put(table_name, digest, data)
|
65
|
+
end
|
66
|
+
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def format_proc
|
71
|
+
proc do |tag, time, record|
|
72
|
+
values = []
|
73
|
+
@key_names.each_with_index do |key, i|
|
74
|
+
value = record[key]
|
75
|
+
values << value
|
76
|
+
end
|
77
|
+
values
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
class CrateClient < CrateRuby::Client
|
83
|
+
|
84
|
+
def initialize(servers = [], opts = {})
|
85
|
+
super(servers, opts)
|
86
|
+
end
|
87
|
+
|
88
|
+
# Copy from https://github.com/crate/crate_ruby/blob/master/lib/crate_ruby/client.rb#L95
|
89
|
+
#
|
90
|
+
# Executes a SQL statement against the Crate HTTP REST endpoint.
|
91
|
+
# @param [String] sql statement to execute
|
92
|
+
# @param [Array] args Array of values used for parameter substitution
|
93
|
+
# @param [Hash] Net::HTTP options (open_timeout, read_timeout)
|
94
|
+
# @return [ResultSet]
|
95
|
+
def execute(sql, args = nil, bulk_args = nil, http_options = {})
|
96
|
+
#@logger.debug sql
|
97
|
+
req = Net::HTTP::Post.new("/_sql", initheader = {'Content-Type' => 'application/json'})
|
98
|
+
body = {"stmt" => sql}
|
99
|
+
body.merge!({'args' => args}) if args
|
100
|
+
body.merge!({'bulk_args' => bulk_args}) if bulk_args
|
101
|
+
req.body = body.to_json
|
102
|
+
response = request(req, http_options)
|
103
|
+
#@logger.debug response.body
|
104
|
+
success = case response.code
|
105
|
+
when /^2\d{2}/
|
106
|
+
ResultSet.new response.body
|
107
|
+
else
|
108
|
+
@logger.info(response.body)
|
109
|
+
raise CrateRuby::CrateError.new(response.body)
|
110
|
+
end
|
111
|
+
success
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
metadata
ADDED
@@ -0,0 +1,95 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-cratedb
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- buom
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-12-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: crate_ruby
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.0.8
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.0.8
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.12.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.12.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
description: Fluent Output Plugin for CrateDB (http://crate.io)
|
56
|
+
email:
|
57
|
+
- me@buom.io
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
64
|
+
- Gemfile
|
65
|
+
- LICENSE.txt
|
66
|
+
- README.md
|
67
|
+
- Rakefile
|
68
|
+
- VERSION
|
69
|
+
- fluent-plugin-cratedb.gemspec
|
70
|
+
- lib/fluent/plugin/out_cratedb.rb
|
71
|
+
homepage: https://github.com/buom/fluent-plugin-cratedb
|
72
|
+
licenses:
|
73
|
+
- Apache-2.0
|
74
|
+
metadata: {}
|
75
|
+
post_install_message:
|
76
|
+
rdoc_options: []
|
77
|
+
require_paths:
|
78
|
+
- lib
|
79
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
requirements: []
|
90
|
+
rubyforge_project:
|
91
|
+
rubygems_version: 2.4.8
|
92
|
+
signing_key:
|
93
|
+
specification_version: 4
|
94
|
+
summary: A plugin for Fluentd
|
95
|
+
test_files: []
|