fluent-plugin-cassandra-driver 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +4 -0
- data/.simplecov +5 -0
- data/Gemfile +12 -0
- data/LICENSE.txt +29 -0
- data/README.md +39 -0
- data/Rakefile +47 -0
- data/VERSION +1 -0
- data/fluent-plugin-cassandra-driver.gemspec +71 -0
- data/lib/fluent/plugin/out_cassandra_driver.rb +109 -0
- data/spec/cassandra_cql_output_spec.rb +158 -0
- data/spec/spec.opts +2 -0
- data/spec/spec_helper.rb +9 -0
- data/spec/support/helpers.rb +56 -0
- metadata +157 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: dc1aec631bec8512c572f9c7b4eef5a2cfcb0a96
|
4
|
+
data.tar.gz: e64678e60780982ad114a504cf55e69735bdd1dc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 552915e4023e251d06f2d8b645a9e495a136a1d7953a34687467271c9c4e6670f5f66ad555abec6af7ba779a3162cc913afddba4894962447189477a7536d7f2
|
7
|
+
data.tar.gz: 22d47d1a6592b80938db0d192cc1a96aeeeb79c4c99f25be54a6646e2a4387bf80e39a78579abde35b38b2f81739f830f02ba6871fbb83c401e0f368d5918fbc
|
data/.rspec
ADDED
data/.simplecov
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
Copyright © 2016 by Yaroslav Lukyanov (c_sharp@mail.ru)
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions
|
6
|
+
are met:
|
7
|
+
|
8
|
+
* Redistributions of source code must retain the above copyright
|
9
|
+
notice, this list of conditions and the following disclaimer.
|
10
|
+
* Redistributions in binary form must reproduce the above copyright
|
11
|
+
notice, this list of conditions and the following disclaimer in
|
12
|
+
the documentation and/or other materials provided with the
|
13
|
+
distribution.
|
14
|
+
* Neither the name of the copyright holder nor the names of its
|
15
|
+
contributors may be used to endorse or promote products derived
|
16
|
+
from this software without specific prior written permission.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
21
|
+
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
22
|
+
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
23
|
+
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
24
|
+
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
25
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27
|
+
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28
|
+
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29
|
+
POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# Cassandra plugin for Fluentd
|
2
|
+
|
3
|
+
Cassandra output plugin for Fluentd.
|
4
|
+
|
5
|
+
Implemented using the Datastax Ruby Driver for Apache Cassandra gem and targets [CQL3](https://docs.datastax.com/en/cql/3.3/)
|
6
|
+
and Cassandra 1.2 - 3.x
|
7
|
+
|
8
|
+
# Installation
|
9
|
+
|
10
|
+
via RubyGems
|
11
|
+
|
12
|
+
fluent-gem install fluent-plugin-cassandra-driver
|
13
|
+
|
14
|
+
# Quick Start
|
15
|
+
|
16
|
+
## Cassandra Configuration
|
17
|
+
# create keyspace (via CQL)
|
18
|
+
CREATE KEYSPACE \"metrics\" WITH strategy_class='org.apache.cassandra.locator.SimpleStrategy' AND strategy_options:replication_factor=1;
|
19
|
+
|
20
|
+
# create table (column family)
|
21
|
+
CREATE TABLE logs (id varchar, ts bigint, payload text, PRIMARY KEY (id, ts)) WITH CLUSTERING ORDER BY (ts DESC);
|
22
|
+
|
23
|
+
# NOTE: schema definition should match that specified in the Fluentd.conf configuration file (see below)
|
24
|
+
|
25
|
+
## Fluentd.conf Configuration
|
26
|
+
<match cassandra.**>
|
27
|
+
type cassandra_driver # fluent output plugin file name (sans fluent_plugin_ prefix)
|
28
|
+
hosts 127.0.0.1 # comma delimited string of hosts
|
29
|
+
keyspace metrics # cassandra keyspace
|
30
|
+
columnfamily logs # cassandra column family
|
31
|
+
ttl 60 # cassandra ttl *optional => default is 0*
|
32
|
+
schema # cassandra column family schema *hash where keys => column names and values => data types* for example: {:id => :string}
|
33
|
+
data_keys # comma delimited string of the fluentd hash's keys
|
34
|
+
pop_data_keys # keep or pop key/values from the fluentd hash when storing it as json
|
35
|
+
</match>
|
36
|
+
|
37
|
+
# Tests
|
38
|
+
|
39
|
+
TODO
|
data/Rakefile
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:default, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts 'Run `bundle install` to install missing gems'
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
Jeweler::Tasks.new do |gem|
|
16
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
17
|
+
gem.name = 'fluent-plugin-cassandra-driver'
|
18
|
+
gem.homepage = 'https://github.com/CSharpRU/fluent-plugin-cassandra-driver'
|
19
|
+
gem.license = 'BSD-3-Clause'
|
20
|
+
gem.summary = 'Fluent output plugin for Cassandra'
|
21
|
+
gem.description = 'Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache Cassandra'
|
22
|
+
gem.email = 'c_sharp@mail.ru'
|
23
|
+
gem.authors = ['Yaroslav Lukyanov']
|
24
|
+
# dependencies defined in Gemfile
|
25
|
+
end
|
26
|
+
Jeweler::RubygemsDotOrgTasks.new
|
27
|
+
|
28
|
+
require 'rdoc/task'
|
29
|
+
Rake::RDocTask.new do |rdoc|
|
30
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
31
|
+
|
32
|
+
rdoc.rdoc_dir = 'rdoc'
|
33
|
+
rdoc.title = "fluent-plugin-cassandra-cql #{version}"
|
34
|
+
rdoc.rdoc_files.include('README*')
|
35
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
36
|
+
end
|
37
|
+
|
38
|
+
# Get spec rake tasks working in RSpec 2.0
|
39
|
+
require 'rspec/core/rake_task'
|
40
|
+
|
41
|
+
desc 'Default: run specs.'
|
42
|
+
task :default => :spec
|
43
|
+
|
44
|
+
desc 'Run specs'
|
45
|
+
RSpec::Core::RakeTask.new do |t|
|
46
|
+
|
47
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
# stub: fluent-plugin-cassandra-driver 0.0.1 ruby lib
|
6
|
+
|
7
|
+
Gem::Specification.new do |s|
|
8
|
+
s.name = "fluent-plugin-cassandra-driver".freeze
|
9
|
+
s.version = "0.0.1"
|
10
|
+
|
11
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
|
+
s.require_paths = ["lib".freeze]
|
13
|
+
s.authors = ["Yaroslav Lukyanov".freeze]
|
14
|
+
s.date = "2016-11-10"
|
15
|
+
s.description = "Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache Cassandra".freeze
|
16
|
+
s.email = "c_sharp@mail.ru".freeze
|
17
|
+
s.extra_rdoc_files = [
|
18
|
+
"LICENSE.txt",
|
19
|
+
"README.md"
|
20
|
+
]
|
21
|
+
s.files = [
|
22
|
+
".rspec",
|
23
|
+
".simplecov",
|
24
|
+
"Gemfile",
|
25
|
+
"LICENSE.txt",
|
26
|
+
"README.md",
|
27
|
+
"Rakefile",
|
28
|
+
"VERSION",
|
29
|
+
"fluent-plugin-cassandra-driver.gemspec",
|
30
|
+
"lib/fluent/plugin/out_cassandra_driver.rb",
|
31
|
+
"spec/cassandra_cql_output_spec.rb",
|
32
|
+
"spec/spec.opts",
|
33
|
+
"spec/spec_helper.rb",
|
34
|
+
"spec/support/helpers.rb"
|
35
|
+
]
|
36
|
+
s.homepage = "https://github.com/CSharpRU/fluent-plugin-cassandra-driver".freeze
|
37
|
+
s.licenses = ["BSD-3-Clause".freeze]
|
38
|
+
s.rubygems_version = "2.6.8".freeze
|
39
|
+
s.summary = "Fluent output plugin for Cassandra".freeze
|
40
|
+
|
41
|
+
if s.respond_to? :specification_version then
|
42
|
+
s.specification_version = 4
|
43
|
+
|
44
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
45
|
+
s.add_runtime_dependency(%q<fluentd>.freeze, [">= 0.12.29"])
|
46
|
+
s.add_runtime_dependency(%q<cassandra-driver>.freeze, [">= 3.0.3"])
|
47
|
+
s.add_development_dependency(%q<rdoc>.freeze, [">= 3.12"])
|
48
|
+
s.add_development_dependency(%q<bundler>.freeze, [">= 1.0.0"])
|
49
|
+
s.add_development_dependency(%q<jeweler>.freeze, [">= 1.8.4"])
|
50
|
+
s.add_development_dependency(%q<rspec>.freeze, [">= 0"])
|
51
|
+
s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
|
52
|
+
else
|
53
|
+
s.add_dependency(%q<fluentd>.freeze, [">= 0.12.29"])
|
54
|
+
s.add_dependency(%q<cassandra-driver>.freeze, [">= 3.0.3"])
|
55
|
+
s.add_dependency(%q<rdoc>.freeze, [">= 3.12"])
|
56
|
+
s.add_dependency(%q<bundler>.freeze, [">= 1.0.0"])
|
57
|
+
s.add_dependency(%q<jeweler>.freeze, [">= 1.8.4"])
|
58
|
+
s.add_dependency(%q<rspec>.freeze, [">= 0"])
|
59
|
+
s.add_dependency(%q<simplecov>.freeze, [">= 0"])
|
60
|
+
end
|
61
|
+
else
|
62
|
+
s.add_dependency(%q<fluentd>.freeze, [">= 0.12.29"])
|
63
|
+
s.add_dependency(%q<cassandra-driver>.freeze, [">= 3.0.3"])
|
64
|
+
s.add_dependency(%q<rdoc>.freeze, [">= 3.12"])
|
65
|
+
s.add_dependency(%q<bundler>.freeze, [">= 1.0.0"])
|
66
|
+
s.add_dependency(%q<jeweler>.freeze, [">= 1.8.4"])
|
67
|
+
s.add_dependency(%q<rspec>.freeze, [">= 0"])
|
68
|
+
s.add_dependency(%q<simplecov>.freeze, [">= 0"])
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'cassandra'
|
2
|
+
require 'msgpack'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class CassandraCqlOutput < BufferedOutput
|
7
|
+
Fluent::Plugin.register_output('cassandra_driver', self)
|
8
|
+
|
9
|
+
config_param :hosts, :string
|
10
|
+
config_param :keyspace, :string
|
11
|
+
config_param :columnfamily, :string
|
12
|
+
config_param :ttl, :integer, :default => 0
|
13
|
+
config_param :schema, :string
|
14
|
+
config_param :data_keys, :string
|
15
|
+
|
16
|
+
# remove keys from the fluentd json event as they're processed
|
17
|
+
# for individual columns?
|
18
|
+
config_param :pop_data_keys, :bool, :default => true
|
19
|
+
|
20
|
+
def session
|
21
|
+
@session ||= get_session(self.hosts, self.keyspace)
|
22
|
+
end
|
23
|
+
|
24
|
+
def configure(conf)
|
25
|
+
super
|
26
|
+
|
27
|
+
# perform validations
|
28
|
+
raise ConfigError, "'Hosts' is required by Cassandra output (ex: localhost, 127.0.0.1, ec2-54-242-141-252.compute-1.amazonaws.com" if self.hosts.nil?
|
29
|
+
raise ConfigError, "'Keyspace' is required by Cassandra output (ex: FluentdLoggers)" if self.keyspace.nil?
|
30
|
+
raise ConfigError, "'ColumnFamily' is required by Cassandra output (ex: events)" if self.columnfamily.nil?
|
31
|
+
raise ConfigError, "'Schema' is required by Cassandra output (ex: id,ts,payload)" if self.schema.nil?
|
32
|
+
raise ConfigError, "'Schema' must contain at least two column names (ex: id,ts,payload)" if self.schema.split(',').count < 2
|
33
|
+
raise ConfigError, "'DataKeys' is required by Cassandra output (ex: tag,created_at,data)" if self.data_keys.nil?
|
34
|
+
|
35
|
+
# convert schema from string to hash
|
36
|
+
# NOTE: ok to use eval b/c this isn't this isn't a user
|
37
|
+
# supplied string
|
38
|
+
self.schema = eval(self.schema)
|
39
|
+
|
40
|
+
# convert data keys from string to array
|
41
|
+
self.data_keys = self.data_keys.split(',')
|
42
|
+
|
43
|
+
# split hosts to array
|
44
|
+
self.hosts = self.hosts.split(',')
|
45
|
+
end
|
46
|
+
|
47
|
+
def start
|
48
|
+
super
|
49
|
+
session
|
50
|
+
end
|
51
|
+
|
52
|
+
def shutdown
|
53
|
+
super
|
54
|
+
@session.close if @session
|
55
|
+
end
|
56
|
+
|
57
|
+
def format(tag, time, record)
|
58
|
+
record.to_msgpack
|
59
|
+
end
|
60
|
+
|
61
|
+
def write(chunk)
|
62
|
+
chunk.msgpack_each { |record|
|
63
|
+
values = build_insert_values_string(self.schema.keys, self.data_keys, record, self.pop_data_keys)
|
64
|
+
|
65
|
+
cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) VALUES (#{values}) USING TTL #{self.ttl}"
|
66
|
+
|
67
|
+
@session.execute(cql)
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def get_session(hosts, keyspace)
|
74
|
+
cluster = ::Cassandra.cluster(hosts: hosts)
|
75
|
+
|
76
|
+
cluster.connect(keyspace)
|
77
|
+
end
|
78
|
+
|
79
|
+
def build_insert_values_string(schema_keys, data_keys, record, pop_data_keys)
|
80
|
+
values = data_keys.map.with_index do |key, index|
|
81
|
+
if pop_data_keys
|
82
|
+
self.schema[schema_keys[index]] == :string ? "'#{record.delete(key)}'" : record.delete(key)
|
83
|
+
else
|
84
|
+
self.schema[schema_keys[index]] == :string ? "'#{record[key]}'" : record[key]
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# if we have one more schema key than data keys,
|
89
|
+
# we can then infer that we should store the event
|
90
|
+
# as a string representation of the corresponding
|
91
|
+
# json object in the last schema column
|
92
|
+
if schema_keys.count == data_keys.count + 1
|
93
|
+
values << if record.count > 0
|
94
|
+
"'#{record.to_json}'"
|
95
|
+
else
|
96
|
+
# by this point, the extra schema column has been
|
97
|
+
# added to insert cql statement, so we must put
|
98
|
+
# something in it
|
99
|
+
# TODO: detect this scenario earlier and don't
|
100
|
+
# specify the column name/value at all
|
101
|
+
# when constructing the cql stmt
|
102
|
+
"''"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
values.join(',')
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
Fluent::Test.setup
|
3
|
+
|
4
|
+
SPEC_COLUMN_FAMILY = "spec_events"
|
5
|
+
DATA_KEYS = "tag,time"
|
6
|
+
|
7
|
+
CONFIG = %[
|
8
|
+
host 127.0.0.1
|
9
|
+
port 9160
|
10
|
+
keyspace FluentdLoggers
|
11
|
+
columnfamily #{SPEC_COLUMN_FAMILY}
|
12
|
+
ttl 0
|
13
|
+
schema {:id => :string, :ts => :bigint, :payload => :string}
|
14
|
+
data_keys #{DATA_KEYS}
|
15
|
+
pop_data_keys true
|
16
|
+
]
|
17
|
+
|
18
|
+
describe Fluent::CassandraCqlOutput do
|
19
|
+
include Helpers
|
20
|
+
|
21
|
+
let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test') }
|
22
|
+
|
23
|
+
after(:each) do
|
24
|
+
d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test')
|
25
|
+
d.configure(CONFIG)
|
26
|
+
d.instance.session.execute("TRUNCATE #{SPEC_COLUMN_FAMILY}")
|
27
|
+
end
|
28
|
+
|
29
|
+
def set_config_value(config, config_name, value)
|
30
|
+
search_text = config.split("\n").map {|text| text if text.strip!.to_s.start_with? config_name.to_s}.compact![0]
|
31
|
+
config.gsub(search_text, "#{config_name} #{value}")
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'configuring' do
|
35
|
+
|
36
|
+
it 'should be properly configured' do
|
37
|
+
driver.configure(CONFIG)
|
38
|
+
driver.tag.should eq('test')
|
39
|
+
driver.instance.host.should eq('127.0.0.1')
|
40
|
+
driver.instance.port.should eq(9160)
|
41
|
+
driver.instance.keyspace.should eq('FluentdLoggers')
|
42
|
+
driver.instance.columnfamily.should eq(SPEC_COLUMN_FAMILY)
|
43
|
+
driver.instance.ttl.should eq(0)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should configure ttl' do
|
47
|
+
ttl = 20
|
48
|
+
driver.configure(set_config_value(CONFIG, :ttl, ttl))
|
49
|
+
driver.instance.ttl.should eq(ttl)
|
50
|
+
end
|
51
|
+
|
52
|
+
describe 'exceptions' do
|
53
|
+
it 'should raise an exception if host is not configured' do
|
54
|
+
expect { driver.configure(CONFIG.gsub("host", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should raise an exception if port is not configured' do
|
58
|
+
expect { driver.configure(CONFIG.gsub("port", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should raise an exception if keyspace is not configured' do
|
62
|
+
expect { driver.configure(CONFIG.gsub("keyspace", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'should raise an exception if columnfamily is not configured' do
|
66
|
+
expect { driver.configure(CONFIG.gsub("columnfamily", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
end # context configuring
|
71
|
+
|
72
|
+
context 'logging' do
|
73
|
+
|
74
|
+
it 'should start' do
|
75
|
+
driver.configure(CONFIG)
|
76
|
+
driver.instance.start
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'should shutdown' do
|
80
|
+
driver.configure(CONFIG)
|
81
|
+
driver.instance.start
|
82
|
+
driver.instance.shutdown
|
83
|
+
end
|
84
|
+
|
85
|
+
it 'should format' do
|
86
|
+
driver.configure(CONFIG)
|
87
|
+
time = Time.now.to_i
|
88
|
+
record = {'tag' => 'test', 'time' => time, 'a' => 1}
|
89
|
+
|
90
|
+
driver.emit(record)
|
91
|
+
driver.expect_format(record.to_msgpack)
|
92
|
+
driver.run
|
93
|
+
end
|
94
|
+
|
95
|
+
context 'writing' do
|
96
|
+
context 'as json' do
|
97
|
+
|
98
|
+
describe 'pop no data keys' do
|
99
|
+
it 'should store json in last column' do
|
100
|
+
driver.configure(set_config_value(CONFIG, :pop_data_keys, false))
|
101
|
+
write(driver, SPEC_COLUMN_FAMILY, false)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
describe 'pop some data keys' do
|
106
|
+
it 'should store json in last last column' do
|
107
|
+
driver.configure(set_config_value(CONFIG, :pop_data_keys, true))
|
108
|
+
write(driver, SPEC_COLUMN_FAMILY, false)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
describe 'pop all data keys' do
|
113
|
+
it 'should store empty string in last column' do
|
114
|
+
driver.configure(CONFIG)
|
115
|
+
write(driver, SPEC_COLUMN_FAMILY, true)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
end # context as json
|
120
|
+
|
121
|
+
context 'as columns' do # no need to test popping of keys b/c it makes no difference
|
122
|
+
|
123
|
+
it 'should write' do
|
124
|
+
config = set_config_value(CONFIG, :data_keys, DATA_KEYS + ',a')
|
125
|
+
config = set_config_value(CONFIG, :pop_data_keys, false)
|
126
|
+
driver.configure(config)
|
127
|
+
write(driver, SPEC_COLUMN_FAMILY, false)
|
128
|
+
end
|
129
|
+
|
130
|
+
end # context as columns
|
131
|
+
|
132
|
+
it 'should not locate event after ttl has expired' do
|
133
|
+
time = Time.now.to_i
|
134
|
+
tag = "ttl_test"
|
135
|
+
ttl = 1 # set ttl to 1 second
|
136
|
+
|
137
|
+
driver.configure(set_config_value(CONFIG, :ttl, ttl))
|
138
|
+
driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
|
139
|
+
driver.run
|
140
|
+
|
141
|
+
# verify record... should return in less than one sec if hitting
|
142
|
+
# cassandra running on localhost
|
143
|
+
events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
|
144
|
+
events.rows.should eq(1)
|
145
|
+
|
146
|
+
# now, sleep long enough for the event to be expired from cassandra
|
147
|
+
sleep(ttl + 1)
|
148
|
+
|
149
|
+
# re-query and verify that no events were returned
|
150
|
+
events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
|
151
|
+
events.rows.should eq(0)
|
152
|
+
end
|
153
|
+
|
154
|
+
end # context writing
|
155
|
+
|
156
|
+
end # context logging
|
157
|
+
|
158
|
+
end # CassandraOutput
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
module Helpers
|
2
|
+
|
3
|
+
def write(driver, column_family_name, tag_and_time_only)
|
4
|
+
tag1 = "test1"
|
5
|
+
tag2 = "test2"
|
6
|
+
time1 = Time.now.to_i
|
7
|
+
time2 = Time.now.to_i + 2
|
8
|
+
|
9
|
+
record1 = {'tag' => tag1, 'time' => time1}
|
10
|
+
record2 = {'tag' => tag2, 'time' => time2}
|
11
|
+
|
12
|
+
unless tag_and_time_only
|
13
|
+
record1.merge!({'a' => 10, 'b' => 'Tesla'})
|
14
|
+
record2.merge!({'a' => 20, 'b' => 'Edison'})
|
15
|
+
end
|
16
|
+
|
17
|
+
# store both records in an array
|
18
|
+
records = [record1, record2]
|
19
|
+
|
20
|
+
driver.emit(records[0])
|
21
|
+
driver.emit(records[1])
|
22
|
+
driver.run # persists to cassandra
|
23
|
+
|
24
|
+
# query cassandra to verify data was correctly persisted
|
25
|
+
row_num = records.count # non-zero based index
|
26
|
+
events = driver.instance.session.execute("SELECT * FROM #{column_family_name}")
|
27
|
+
events.rows.should eq(records.count)
|
28
|
+
events.fetch do | event | # events should be sorted desc by tag, then time
|
29
|
+
row_num -= 1 # zero-based index
|
30
|
+
|
31
|
+
record = records[row_num]
|
32
|
+
db_hash = event.to_hash
|
33
|
+
|
34
|
+
# need to take in account that we've popped both tag and time
|
35
|
+
# from the payload data when we saved it
|
36
|
+
if driver.instance.pop_data_keys
|
37
|
+
db_hash['id'].should eq(record.delete('tag'))
|
38
|
+
db_hash['ts'].should eq(record.delete('time'))
|
39
|
+
else
|
40
|
+
db_hash['id'].should eq(record['tag'])
|
41
|
+
db_hash['ts'].should eq(record['time'])
|
42
|
+
end
|
43
|
+
|
44
|
+
if driver.instance.schema.keys.count == driver.instance.data_keys.count + 1 # store as json
|
45
|
+
if record.count > 0
|
46
|
+
db_hash['payload'].should eq(record.to_json)
|
47
|
+
else
|
48
|
+
db_hash['payload'].should eq('')
|
49
|
+
end
|
50
|
+
else
|
51
|
+
db_hash['payload'].should eq(record[record.keys[db_hash.keys.index('payload')]])
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
metadata
ADDED
@@ -0,0 +1,157 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-cassandra-driver
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yaroslav Lukyanov
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-11-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.12.29
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.12.29
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: cassandra-driver
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 3.0.3
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 3.0.3
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rdoc
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.12'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.12'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.0.0
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.0.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: jeweler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 1.8.4
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.8.4
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: simplecov
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
description: Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache
|
112
|
+
Cassandra
|
113
|
+
email: c_sharp@mail.ru
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files:
|
117
|
+
- LICENSE.txt
|
118
|
+
- README.md
|
119
|
+
files:
|
120
|
+
- ".rspec"
|
121
|
+
- ".simplecov"
|
122
|
+
- Gemfile
|
123
|
+
- LICENSE.txt
|
124
|
+
- README.md
|
125
|
+
- Rakefile
|
126
|
+
- VERSION
|
127
|
+
- fluent-plugin-cassandra-driver.gemspec
|
128
|
+
- lib/fluent/plugin/out_cassandra_driver.rb
|
129
|
+
- spec/cassandra_cql_output_spec.rb
|
130
|
+
- spec/spec.opts
|
131
|
+
- spec/spec_helper.rb
|
132
|
+
- spec/support/helpers.rb
|
133
|
+
homepage: https://github.com/CSharpRU/fluent-plugin-cassandra-driver
|
134
|
+
licenses:
|
135
|
+
- BSD-3-Clause
|
136
|
+
metadata: {}
|
137
|
+
post_install_message:
|
138
|
+
rdoc_options: []
|
139
|
+
require_paths:
|
140
|
+
- lib
|
141
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - ">="
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: '0'
|
151
|
+
requirements: []
|
152
|
+
rubyforge_project:
|
153
|
+
rubygems_version: 2.6.8
|
154
|
+
signing_key:
|
155
|
+
specification_version: 4
|
156
|
+
summary: Fluent output plugin for Cassandra
|
157
|
+
test_files: []
|