fluent-plugin-cassandra-driver 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.simplecov +5 -0
- data/Gemfile +12 -0
- data/LICENSE.txt +29 -0
- data/README.md +39 -0
- data/Rakefile +47 -0
- data/VERSION +1 -0
- data/fluent-plugin-cassandra-driver.gemspec +71 -0
- data/lib/fluent/plugin/out_cassandra_driver.rb +109 -0
- data/spec/cassandra_cql_output_spec.rb +158 -0
- data/spec/spec.opts +2 -0
- data/spec/spec_helper.rb +9 -0
- data/spec/support/helpers.rb +56 -0
- metadata +157 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: dc1aec631bec8512c572f9c7b4eef5a2cfcb0a96
|
4
|
+
data.tar.gz: e64678e60780982ad114a504cf55e69735bdd1dc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 552915e4023e251d06f2d8b645a9e495a136a1d7953a34687467271c9c4e6670f5f66ad555abec6af7ba779a3162cc913afddba4894962447189477a7536d7f2
|
7
|
+
data.tar.gz: 22d47d1a6592b80938db0d192cc1a96aeeeb79c4c99f25be54a6646e2a4387bf80e39a78579abde35b38b2f81739f830f02ba6871fbb83c401e0f368d5918fbc
|
data/.rspec
ADDED
data/.simplecov
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
Copyright © 2016 by Yaroslav Lukyanov (c_sharp@mail.ru)
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions
|
6
|
+
are met:
|
7
|
+
|
8
|
+
* Redistributions of source code must retain the above copyright
|
9
|
+
notice, this list of conditions and the following disclaimer.
|
10
|
+
* Redistributions in binary form must reproduce the above copyright
|
11
|
+
notice, this list of conditions and the following disclaimer in
|
12
|
+
the documentation and/or other materials provided with the
|
13
|
+
distribution.
|
14
|
+
* Neither the name of the copyright holder nor the names of its
|
15
|
+
contributors may be used to endorse or promote products derived
|
16
|
+
from this software without specific prior written permission.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
21
|
+
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
22
|
+
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
23
|
+
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
24
|
+
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
25
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27
|
+
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28
|
+
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29
|
+
POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# Cassandra plugin for Fluentd
|
2
|
+
|
3
|
+
Cassandra output plugin for Fluentd.
|
4
|
+
|
5
|
+
Implemented using the Datastax Ruby Driver for Apache Cassandra gem and targets [CQL3](https://docs.datastax.com/en/cql/3.3/)
|
6
|
+
and Cassandra 1.2 - 3.x
|
7
|
+
|
8
|
+
# Installation
|
9
|
+
|
10
|
+
via RubyGems
|
11
|
+
|
12
|
+
fluent-gem install fluent-plugin-cassandra-driver
|
13
|
+
|
14
|
+
# Quick Start
|
15
|
+
|
16
|
+
## Cassandra Configuration
|
17
|
+
# create keyspace (via CQL)
|
18
|
+
CREATE KEYSPACE \"metrics\" WITH strategy_class='org.apache.cassandra.locator.SimpleStrategy' AND strategy_options:replication_factor=1;
|
19
|
+
|
20
|
+
# create table (column family)
|
21
|
+
CREATE TABLE logs (id varchar, ts bigint, payload text, PRIMARY KEY (id, ts)) WITH CLUSTERING ORDER BY (ts DESC);
|
22
|
+
|
23
|
+
# NOTE: schema definition should match that specified in the Fluentd.conf configuration file (see below)
|
24
|
+
|
25
|
+
## Fluentd.conf Configuration
|
26
|
+
<match cassandra.**>
|
27
|
+
type cassandra_driver # fluent output plugin file name (sans fluent_plugin_ prefix)
|
28
|
+
hosts 127.0.0.1 # comma delimited string of hosts
|
29
|
+
keyspace metrics # cassandra keyspace
|
30
|
+
columnfamily logs # cassandra column family
|
31
|
+
ttl 60 # cassandra ttl *optional => default is 0*
|
32
|
+
schema # cassandra column family schema *hash where keys => column names and values => data types* for example: {:id => :string}
|
33
|
+
data_keys # comma delimited string of the fluentd hash's keys
|
34
|
+
pop_data_keys # keep or pop key/values from the fluentd hash when storing it as json
|
35
|
+
</match>
|
36
|
+
|
37
|
+
# Tests
|
38
|
+
|
39
|
+
TODO
|
data/Rakefile
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:default, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts 'Run `bundle install` to install missing gems'
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
Jeweler::Tasks.new do |gem|
|
16
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
17
|
+
gem.name = 'fluent-plugin-cassandra-driver'
|
18
|
+
gem.homepage = 'https://github.com/CSharpRU/fluent-plugin-cassandra-driver'
|
19
|
+
gem.license = 'BSD-3-Clause'
|
20
|
+
gem.summary = 'Fluent output plugin for Cassandra'
|
21
|
+
gem.description = 'Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache Cassandra'
|
22
|
+
gem.email = 'c_sharp@mail.ru'
|
23
|
+
gem.authors = ['Yaroslav Lukyanov']
|
24
|
+
# dependencies defined in Gemfile
|
25
|
+
end
|
26
|
+
Jeweler::RubygemsDotOrgTasks.new
|
27
|
+
|
28
|
+
require 'rdoc/task'
|
29
|
+
Rake::RDocTask.new do |rdoc|
|
30
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
31
|
+
|
32
|
+
rdoc.rdoc_dir = 'rdoc'
|
33
|
+
rdoc.title = "fluent-plugin-cassandra-cql #{version}"
|
34
|
+
rdoc.rdoc_files.include('README*')
|
35
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
36
|
+
end
|
37
|
+
|
38
|
+
# Get spec rake tasks working in RSpec 2.0
|
39
|
+
require 'rspec/core/rake_task'
|
40
|
+
|
41
|
+
desc 'Default: run specs.'
|
42
|
+
task :default => :spec
|
43
|
+
|
44
|
+
desc 'Run specs'
|
45
|
+
RSpec::Core::RakeTask.new do |t|
|
46
|
+
|
47
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
# stub: fluent-plugin-cassandra-driver 0.0.1 ruby lib
|
6
|
+
|
7
|
+
Gem::Specification.new do |s|
|
8
|
+
s.name = "fluent-plugin-cassandra-driver".freeze
|
9
|
+
s.version = "0.0.1"
|
10
|
+
|
11
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
|
+
s.require_paths = ["lib".freeze]
|
13
|
+
s.authors = ["Yaroslav Lukyanov".freeze]
|
14
|
+
s.date = "2016-11-10"
|
15
|
+
s.description = "Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache Cassandra".freeze
|
16
|
+
s.email = "c_sharp@mail.ru".freeze
|
17
|
+
s.extra_rdoc_files = [
|
18
|
+
"LICENSE.txt",
|
19
|
+
"README.md"
|
20
|
+
]
|
21
|
+
s.files = [
|
22
|
+
".rspec",
|
23
|
+
".simplecov",
|
24
|
+
"Gemfile",
|
25
|
+
"LICENSE.txt",
|
26
|
+
"README.md",
|
27
|
+
"Rakefile",
|
28
|
+
"VERSION",
|
29
|
+
"fluent-plugin-cassandra-driver.gemspec",
|
30
|
+
"lib/fluent/plugin/out_cassandra_driver.rb",
|
31
|
+
"spec/cassandra_cql_output_spec.rb",
|
32
|
+
"spec/spec.opts",
|
33
|
+
"spec/spec_helper.rb",
|
34
|
+
"spec/support/helpers.rb"
|
35
|
+
]
|
36
|
+
s.homepage = "https://github.com/CSharpRU/fluent-plugin-cassandra-driver".freeze
|
37
|
+
s.licenses = ["BSD-3-Clause".freeze]
|
38
|
+
s.rubygems_version = "2.6.8".freeze
|
39
|
+
s.summary = "Fluent output plugin for Cassandra".freeze
|
40
|
+
|
41
|
+
if s.respond_to? :specification_version then
|
42
|
+
s.specification_version = 4
|
43
|
+
|
44
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
45
|
+
s.add_runtime_dependency(%q<fluentd>.freeze, [">= 0.12.29"])
|
46
|
+
s.add_runtime_dependency(%q<cassandra-driver>.freeze, [">= 3.0.3"])
|
47
|
+
s.add_development_dependency(%q<rdoc>.freeze, [">= 3.12"])
|
48
|
+
s.add_development_dependency(%q<bundler>.freeze, [">= 1.0.0"])
|
49
|
+
s.add_development_dependency(%q<jeweler>.freeze, [">= 1.8.4"])
|
50
|
+
s.add_development_dependency(%q<rspec>.freeze, [">= 0"])
|
51
|
+
s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
|
52
|
+
else
|
53
|
+
s.add_dependency(%q<fluentd>.freeze, [">= 0.12.29"])
|
54
|
+
s.add_dependency(%q<cassandra-driver>.freeze, [">= 3.0.3"])
|
55
|
+
s.add_dependency(%q<rdoc>.freeze, [">= 3.12"])
|
56
|
+
s.add_dependency(%q<bundler>.freeze, [">= 1.0.0"])
|
57
|
+
s.add_dependency(%q<jeweler>.freeze, [">= 1.8.4"])
|
58
|
+
s.add_dependency(%q<rspec>.freeze, [">= 0"])
|
59
|
+
s.add_dependency(%q<simplecov>.freeze, [">= 0"])
|
60
|
+
end
|
61
|
+
else
|
62
|
+
s.add_dependency(%q<fluentd>.freeze, [">= 0.12.29"])
|
63
|
+
s.add_dependency(%q<cassandra-driver>.freeze, [">= 3.0.3"])
|
64
|
+
s.add_dependency(%q<rdoc>.freeze, [">= 3.12"])
|
65
|
+
s.add_dependency(%q<bundler>.freeze, [">= 1.0.0"])
|
66
|
+
s.add_dependency(%q<jeweler>.freeze, [">= 1.8.4"])
|
67
|
+
s.add_dependency(%q<rspec>.freeze, [">= 0"])
|
68
|
+
s.add_dependency(%q<simplecov>.freeze, [">= 0"])
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'cassandra'
|
2
|
+
require 'msgpack'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class CassandraCqlOutput < BufferedOutput
|
7
|
+
Fluent::Plugin.register_output('cassandra_driver', self)
|
8
|
+
|
9
|
+
config_param :hosts, :string
|
10
|
+
config_param :keyspace, :string
|
11
|
+
config_param :columnfamily, :string
|
12
|
+
config_param :ttl, :integer, :default => 0
|
13
|
+
config_param :schema, :string
|
14
|
+
config_param :data_keys, :string
|
15
|
+
|
16
|
+
# remove keys from the fluentd json event as they're processed
|
17
|
+
# for individual columns?
|
18
|
+
config_param :pop_data_keys, :bool, :default => true
|
19
|
+
|
20
|
+
def session
|
21
|
+
@session ||= get_session(self.hosts, self.keyspace)
|
22
|
+
end
|
23
|
+
|
24
|
+
def configure(conf)
|
25
|
+
super
|
26
|
+
|
27
|
+
# perform validations
|
28
|
+
raise ConfigError, "'Hosts' is required by Cassandra output (ex: localhost, 127.0.0.1, ec2-54-242-141-252.compute-1.amazonaws.com" if self.hosts.nil?
|
29
|
+
raise ConfigError, "'Keyspace' is required by Cassandra output (ex: FluentdLoggers)" if self.keyspace.nil?
|
30
|
+
raise ConfigError, "'ColumnFamily' is required by Cassandra output (ex: events)" if self.columnfamily.nil?
|
31
|
+
raise ConfigError, "'Schema' is required by Cassandra output (ex: id,ts,payload)" if self.schema.nil?
|
32
|
+
raise ConfigError, "'Schema' must contain at least two column names (ex: id,ts,payload)" if self.schema.split(',').count < 2
|
33
|
+
raise ConfigError, "'DataKeys' is required by Cassandra output (ex: tag,created_at,data)" if self.data_keys.nil?
|
34
|
+
|
35
|
+
# convert schema from string to hash
|
36
|
+
# NOTE: ok to use eval b/c this isn't this isn't a user
|
37
|
+
# supplied string
|
38
|
+
self.schema = eval(self.schema)
|
39
|
+
|
40
|
+
# convert data keys from string to array
|
41
|
+
self.data_keys = self.data_keys.split(',')
|
42
|
+
|
43
|
+
# split hosts to array
|
44
|
+
self.hosts = self.hosts.split(',')
|
45
|
+
end
|
46
|
+
|
47
|
+
def start
|
48
|
+
super
|
49
|
+
session
|
50
|
+
end
|
51
|
+
|
52
|
+
def shutdown
|
53
|
+
super
|
54
|
+
@session.close if @session
|
55
|
+
end
|
56
|
+
|
57
|
+
def format(tag, time, record)
|
58
|
+
record.to_msgpack
|
59
|
+
end
|
60
|
+
|
61
|
+
def write(chunk)
|
62
|
+
chunk.msgpack_each { |record|
|
63
|
+
values = build_insert_values_string(self.schema.keys, self.data_keys, record, self.pop_data_keys)
|
64
|
+
|
65
|
+
cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) VALUES (#{values}) USING TTL #{self.ttl}"
|
66
|
+
|
67
|
+
@session.execute(cql)
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def get_session(hosts, keyspace)
|
74
|
+
cluster = ::Cassandra.cluster(hosts: hosts)
|
75
|
+
|
76
|
+
cluster.connect(keyspace)
|
77
|
+
end
|
78
|
+
|
79
|
+
def build_insert_values_string(schema_keys, data_keys, record, pop_data_keys)
|
80
|
+
values = data_keys.map.with_index do |key, index|
|
81
|
+
if pop_data_keys
|
82
|
+
self.schema[schema_keys[index]] == :string ? "'#{record.delete(key)}'" : record.delete(key)
|
83
|
+
else
|
84
|
+
self.schema[schema_keys[index]] == :string ? "'#{record[key]}'" : record[key]
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# if we have one more schema key than data keys,
|
89
|
+
# we can then infer that we should store the event
|
90
|
+
# as a string representation of the corresponding
|
91
|
+
# json object in the last schema column
|
92
|
+
if schema_keys.count == data_keys.count + 1
|
93
|
+
values << if record.count > 0
|
94
|
+
"'#{record.to_json}'"
|
95
|
+
else
|
96
|
+
# by this point, the extra schema column has been
|
97
|
+
# added to insert cql statement, so we must put
|
98
|
+
# something in it
|
99
|
+
# TODO: detect this scenario earlier and don't
|
100
|
+
# specify the column name/value at all
|
101
|
+
# when constructing the cql stmt
|
102
|
+
"''"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
values.join(',')
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
Fluent::Test.setup
|
3
|
+
|
4
|
+
SPEC_COLUMN_FAMILY = "spec_events"
|
5
|
+
DATA_KEYS = "tag,time"
|
6
|
+
|
7
|
+
CONFIG = %[
|
8
|
+
host 127.0.0.1
|
9
|
+
port 9160
|
10
|
+
keyspace FluentdLoggers
|
11
|
+
columnfamily #{SPEC_COLUMN_FAMILY}
|
12
|
+
ttl 0
|
13
|
+
schema {:id => :string, :ts => :bigint, :payload => :string}
|
14
|
+
data_keys #{DATA_KEYS}
|
15
|
+
pop_data_keys true
|
16
|
+
]
|
17
|
+
|
18
|
+
describe Fluent::CassandraCqlOutput do
|
19
|
+
include Helpers
|
20
|
+
|
21
|
+
let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test') }
|
22
|
+
|
23
|
+
after(:each) do
|
24
|
+
d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test')
|
25
|
+
d.configure(CONFIG)
|
26
|
+
d.instance.session.execute("TRUNCATE #{SPEC_COLUMN_FAMILY}")
|
27
|
+
end
|
28
|
+
|
29
|
+
def set_config_value(config, config_name, value)
|
30
|
+
search_text = config.split("\n").map {|text| text if text.strip!.to_s.start_with? config_name.to_s}.compact![0]
|
31
|
+
config.gsub(search_text, "#{config_name} #{value}")
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'configuring' do
|
35
|
+
|
36
|
+
it 'should be properly configured' do
|
37
|
+
driver.configure(CONFIG)
|
38
|
+
driver.tag.should eq('test')
|
39
|
+
driver.instance.host.should eq('127.0.0.1')
|
40
|
+
driver.instance.port.should eq(9160)
|
41
|
+
driver.instance.keyspace.should eq('FluentdLoggers')
|
42
|
+
driver.instance.columnfamily.should eq(SPEC_COLUMN_FAMILY)
|
43
|
+
driver.instance.ttl.should eq(0)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should configure ttl' do
|
47
|
+
ttl = 20
|
48
|
+
driver.configure(set_config_value(CONFIG, :ttl, ttl))
|
49
|
+
driver.instance.ttl.should eq(ttl)
|
50
|
+
end
|
51
|
+
|
52
|
+
describe 'exceptions' do
|
53
|
+
it 'should raise an exception if host is not configured' do
|
54
|
+
expect { driver.configure(CONFIG.gsub("host", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should raise an exception if port is not configured' do
|
58
|
+
expect { driver.configure(CONFIG.gsub("port", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should raise an exception if keyspace is not configured' do
|
62
|
+
expect { driver.configure(CONFIG.gsub("keyspace", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'should raise an exception if columnfamily is not configured' do
|
66
|
+
expect { driver.configure(CONFIG.gsub("columnfamily", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
end # context configuring
|
71
|
+
|
72
|
+
context 'logging' do
|
73
|
+
|
74
|
+
it 'should start' do
|
75
|
+
driver.configure(CONFIG)
|
76
|
+
driver.instance.start
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'should shutdown' do
|
80
|
+
driver.configure(CONFIG)
|
81
|
+
driver.instance.start
|
82
|
+
driver.instance.shutdown
|
83
|
+
end
|
84
|
+
|
85
|
+
it 'should format' do
|
86
|
+
driver.configure(CONFIG)
|
87
|
+
time = Time.now.to_i
|
88
|
+
record = {'tag' => 'test', 'time' => time, 'a' => 1}
|
89
|
+
|
90
|
+
driver.emit(record)
|
91
|
+
driver.expect_format(record.to_msgpack)
|
92
|
+
driver.run
|
93
|
+
end
|
94
|
+
|
95
|
+
context 'writing' do
|
96
|
+
context 'as json' do
|
97
|
+
|
98
|
+
describe 'pop no data keys' do
|
99
|
+
it 'should store json in last column' do
|
100
|
+
driver.configure(set_config_value(CONFIG, :pop_data_keys, false))
|
101
|
+
write(driver, SPEC_COLUMN_FAMILY, false)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
describe 'pop some data keys' do
|
106
|
+
it 'should store json in last last column' do
|
107
|
+
driver.configure(set_config_value(CONFIG, :pop_data_keys, true))
|
108
|
+
write(driver, SPEC_COLUMN_FAMILY, false)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
describe 'pop all data keys' do
|
113
|
+
it 'should store empty string in last column' do
|
114
|
+
driver.configure(CONFIG)
|
115
|
+
write(driver, SPEC_COLUMN_FAMILY, true)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
end # context as json
|
120
|
+
|
121
|
+
context 'as columns' do # no need to test popping of keys b/c it makes no difference
|
122
|
+
|
123
|
+
it 'should write' do
|
124
|
+
config = set_config_value(CONFIG, :data_keys, DATA_KEYS + ',a')
|
125
|
+
config = set_config_value(CONFIG, :pop_data_keys, false)
|
126
|
+
driver.configure(config)
|
127
|
+
write(driver, SPEC_COLUMN_FAMILY, false)
|
128
|
+
end
|
129
|
+
|
130
|
+
end # context as columns
|
131
|
+
|
132
|
+
it 'should not locate event after ttl has expired' do
|
133
|
+
time = Time.now.to_i
|
134
|
+
tag = "ttl_test"
|
135
|
+
ttl = 1 # set ttl to 1 second
|
136
|
+
|
137
|
+
driver.configure(set_config_value(CONFIG, :ttl, ttl))
|
138
|
+
driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
|
139
|
+
driver.run
|
140
|
+
|
141
|
+
# verify record... should return in less than one sec if hitting
|
142
|
+
# cassandra running on localhost
|
143
|
+
events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
|
144
|
+
events.rows.should eq(1)
|
145
|
+
|
146
|
+
# now, sleep long enough for the event to be expired from cassandra
|
147
|
+
sleep(ttl + 1)
|
148
|
+
|
149
|
+
# re-query and verify that no events were returned
|
150
|
+
events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
|
151
|
+
events.rows.should eq(0)
|
152
|
+
end
|
153
|
+
|
154
|
+
end # context writing
|
155
|
+
|
156
|
+
end # context logging
|
157
|
+
|
158
|
+
end # CassandraOutput
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
module Helpers
|
2
|
+
|
3
|
+
def write(driver, column_family_name, tag_and_time_only)
|
4
|
+
tag1 = "test1"
|
5
|
+
tag2 = "test2"
|
6
|
+
time1 = Time.now.to_i
|
7
|
+
time2 = Time.now.to_i + 2
|
8
|
+
|
9
|
+
record1 = {'tag' => tag1, 'time' => time1}
|
10
|
+
record2 = {'tag' => tag2, 'time' => time2}
|
11
|
+
|
12
|
+
unless tag_and_time_only
|
13
|
+
record1.merge!({'a' => 10, 'b' => 'Tesla'})
|
14
|
+
record2.merge!({'a' => 20, 'b' => 'Edison'})
|
15
|
+
end
|
16
|
+
|
17
|
+
# store both records in an array
|
18
|
+
records = [record1, record2]
|
19
|
+
|
20
|
+
driver.emit(records[0])
|
21
|
+
driver.emit(records[1])
|
22
|
+
driver.run # persists to cassandra
|
23
|
+
|
24
|
+
# query cassandra to verify data was correctly persisted
|
25
|
+
row_num = records.count # non-zero based index
|
26
|
+
events = driver.instance.session.execute("SELECT * FROM #{column_family_name}")
|
27
|
+
events.rows.should eq(records.count)
|
28
|
+
events.fetch do | event | # events should be sorted desc by tag, then time
|
29
|
+
row_num -= 1 # zero-based index
|
30
|
+
|
31
|
+
record = records[row_num]
|
32
|
+
db_hash = event.to_hash
|
33
|
+
|
34
|
+
# need to take in account that we've popped both tag and time
|
35
|
+
# from the payload data when we saved it
|
36
|
+
if driver.instance.pop_data_keys
|
37
|
+
db_hash['id'].should eq(record.delete('tag'))
|
38
|
+
db_hash['ts'].should eq(record.delete('time'))
|
39
|
+
else
|
40
|
+
db_hash['id'].should eq(record['tag'])
|
41
|
+
db_hash['ts'].should eq(record['time'])
|
42
|
+
end
|
43
|
+
|
44
|
+
if driver.instance.schema.keys.count == driver.instance.data_keys.count + 1 # store as json
|
45
|
+
if record.count > 0
|
46
|
+
db_hash['payload'].should eq(record.to_json)
|
47
|
+
else
|
48
|
+
db_hash['payload'].should eq('')
|
49
|
+
end
|
50
|
+
else
|
51
|
+
db_hash['payload'].should eq(record[record.keys[db_hash.keys.index('payload')]])
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
metadata
ADDED
@@ -0,0 +1,157 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-cassandra-driver
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yaroslav Lukyanov
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-11-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.12.29
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.12.29
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: cassandra-driver
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 3.0.3
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 3.0.3
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rdoc
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.12'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.12'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.0.0
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.0.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: jeweler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 1.8.4
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.8.4
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: simplecov
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
description: Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache
|
112
|
+
Cassandra
|
113
|
+
email: c_sharp@mail.ru
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files:
|
117
|
+
- LICENSE.txt
|
118
|
+
- README.md
|
119
|
+
files:
|
120
|
+
- ".rspec"
|
121
|
+
- ".simplecov"
|
122
|
+
- Gemfile
|
123
|
+
- LICENSE.txt
|
124
|
+
- README.md
|
125
|
+
- Rakefile
|
126
|
+
- VERSION
|
127
|
+
- fluent-plugin-cassandra-driver.gemspec
|
128
|
+
- lib/fluent/plugin/out_cassandra_driver.rb
|
129
|
+
- spec/cassandra_cql_output_spec.rb
|
130
|
+
- spec/spec.opts
|
131
|
+
- spec/spec_helper.rb
|
132
|
+
- spec/support/helpers.rb
|
133
|
+
homepage: https://github.com/CSharpRU/fluent-plugin-cassandra-driver
|
134
|
+
licenses:
|
135
|
+
- BSD-3-Clause
|
136
|
+
metadata: {}
|
137
|
+
post_install_message:
|
138
|
+
rdoc_options: []
|
139
|
+
require_paths:
|
140
|
+
- lib
|
141
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - ">="
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: '0'
|
151
|
+
requirements: []
|
152
|
+
rubyforge_project:
|
153
|
+
rubygems_version: 2.6.8
|
154
|
+
signing_key:
|
155
|
+
specification_version: 4
|
156
|
+
summary: Fluent output plugin for Cassandra
|
157
|
+
test_files: []
|