fluent-plugin-cassandra-driver 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +6 -2
- data/VERSION +1 -1
- data/fluent-plugin-cassandra-driver.gemspec +3 -5
- data/lib/fluent/plugin/out_cassandra_driver.rb +7 -1
- metadata +1 -3
- data/spec/cassandra_cql_output_spec.rb +0 -158
- data/spec/support/helpers.rb +0 -56
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ac22674557433f0ea85c00863cee8ac7c59be11
|
4
|
+
data.tar.gz: 7e2f89d4571c7cc7801e6386458212ea3a2f6d33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5c9d9fac0898971fac9967c015af167e0155dd0fd8cffe4fa12a1c1005c94c5e44c6000609a428fe45e1c9d5ebe9d8d64b666577830d8734a776b44040e1c0d6
|
7
|
+
data.tar.gz: 90128ed2a057a4618d1ce3b9104a8f635daaf7a4aeeb69e9b46d9e6dc6dcdeb763cb86238803b11ec557f0e8588270a088431ae6d008d52d8dc5e2ab533c3c56
|
data/Rakefile
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'rubygems'
|
4
4
|
require 'bundler'
|
5
|
+
|
5
6
|
begin
|
6
7
|
Bundler.setup(:default, :development)
|
7
8
|
rescue Bundler::BundlerError => e
|
@@ -9,9 +10,10 @@ rescue Bundler::BundlerError => e
|
|
9
10
|
$stderr.puts 'Run `bundle install` to install missing gems'
|
10
11
|
exit e.status_code
|
11
12
|
end
|
12
|
-
require 'rake'
|
13
13
|
|
14
|
+
require 'rake'
|
14
15
|
require 'jeweler'
|
16
|
+
|
15
17
|
Jeweler::Tasks.new do |gem|
|
16
18
|
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
17
19
|
gem.name = 'fluent-plugin-cassandra-driver'
|
@@ -23,14 +25,16 @@ Jeweler::Tasks.new do |gem|
|
|
23
25
|
gem.authors = ['Yaroslav Lukyanov']
|
24
26
|
# dependencies defined in Gemfile
|
25
27
|
end
|
28
|
+
|
26
29
|
Jeweler::RubygemsDotOrgTasks.new
|
27
30
|
|
28
31
|
require 'rdoc/task'
|
32
|
+
|
29
33
|
Rake::RDocTask.new do |rdoc|
|
30
34
|
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
31
35
|
|
32
36
|
rdoc.rdoc_dir = 'rdoc'
|
33
|
-
rdoc.title = "fluent-plugin-cassandra-
|
37
|
+
rdoc.title = "fluent-plugin-cassandra-driver #{version}"
|
34
38
|
rdoc.rdoc_files.include('README*')
|
35
39
|
rdoc.rdoc_files.include('lib/**/*.rb')
|
36
40
|
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.2
|
@@ -2,11 +2,11 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: fluent-plugin-cassandra-driver 0.0.
|
5
|
+
# stub: fluent-plugin-cassandra-driver 0.0.2 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "fluent-plugin-cassandra-driver".freeze
|
9
|
-
s.version = "0.0.
|
9
|
+
s.version = "0.0.2"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
@@ -28,10 +28,8 @@ Gem::Specification.new do |s|
|
|
28
28
|
"VERSION",
|
29
29
|
"fluent-plugin-cassandra-driver.gemspec",
|
30
30
|
"lib/fluent/plugin/out_cassandra_driver.rb",
|
31
|
-
"spec/cassandra_cql_output_spec.rb",
|
32
31
|
"spec/spec.opts",
|
33
|
-
"spec/spec_helper.rb"
|
34
|
-
"spec/support/helpers.rb"
|
32
|
+
"spec/spec_helper.rb"
|
35
33
|
]
|
36
34
|
s.homepage = "https://github.com/CSharpRU/fluent-plugin-cassandra-driver".freeze
|
37
35
|
s.licenses = ["BSD-3-Clause".freeze]
|
@@ -60,11 +60,17 @@ module Fluent
|
|
60
60
|
|
61
61
|
def write(chunk)
|
62
62
|
chunk.msgpack_each { |record|
|
63
|
+
$log.info "Sending a new record to Cassandra: #{record.to_json}"
|
64
|
+
|
63
65
|
values = build_insert_values_string(self.schema.keys, self.data_keys, record, self.pop_data_keys)
|
64
66
|
|
65
67
|
cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) VALUES (#{values}) USING TTL #{self.ttl}"
|
66
68
|
|
67
|
-
|
69
|
+
begin
|
70
|
+
@session.execute(cql)
|
71
|
+
rescue Exception => e
|
72
|
+
$log.error "Cannot send record to Cassandra: #{e.message}\nTrace: #{e.backtrace.to_s}"
|
73
|
+
end
|
68
74
|
}
|
69
75
|
end
|
70
76
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-cassandra-driver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yaroslav Lukyanov
|
@@ -126,10 +126,8 @@ files:
|
|
126
126
|
- VERSION
|
127
127
|
- fluent-plugin-cassandra-driver.gemspec
|
128
128
|
- lib/fluent/plugin/out_cassandra_driver.rb
|
129
|
-
- spec/cassandra_cql_output_spec.rb
|
130
129
|
- spec/spec.opts
|
131
130
|
- spec/spec_helper.rb
|
132
|
-
- spec/support/helpers.rb
|
133
131
|
homepage: https://github.com/CSharpRU/fluent-plugin-cassandra-driver
|
134
132
|
licenses:
|
135
133
|
- BSD-3-Clause
|
@@ -1,158 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
Fluent::Test.setup
|
3
|
-
|
4
|
-
SPEC_COLUMN_FAMILY = "spec_events"
|
5
|
-
DATA_KEYS = "tag,time"
|
6
|
-
|
7
|
-
CONFIG = %[
|
8
|
-
host 127.0.0.1
|
9
|
-
port 9160
|
10
|
-
keyspace FluentdLoggers
|
11
|
-
columnfamily #{SPEC_COLUMN_FAMILY}
|
12
|
-
ttl 0
|
13
|
-
schema {:id => :string, :ts => :bigint, :payload => :string}
|
14
|
-
data_keys #{DATA_KEYS}
|
15
|
-
pop_data_keys true
|
16
|
-
]
|
17
|
-
|
18
|
-
describe Fluent::CassandraCqlOutput do
|
19
|
-
include Helpers
|
20
|
-
|
21
|
-
let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test') }
|
22
|
-
|
23
|
-
after(:each) do
|
24
|
-
d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test')
|
25
|
-
d.configure(CONFIG)
|
26
|
-
d.instance.session.execute("TRUNCATE #{SPEC_COLUMN_FAMILY}")
|
27
|
-
end
|
28
|
-
|
29
|
-
def set_config_value(config, config_name, value)
|
30
|
-
search_text = config.split("\n").map {|text| text if text.strip!.to_s.start_with? config_name.to_s}.compact![0]
|
31
|
-
config.gsub(search_text, "#{config_name} #{value}")
|
32
|
-
end
|
33
|
-
|
34
|
-
context 'configuring' do
|
35
|
-
|
36
|
-
it 'should be properly configured' do
|
37
|
-
driver.configure(CONFIG)
|
38
|
-
driver.tag.should eq('test')
|
39
|
-
driver.instance.host.should eq('127.0.0.1')
|
40
|
-
driver.instance.port.should eq(9160)
|
41
|
-
driver.instance.keyspace.should eq('FluentdLoggers')
|
42
|
-
driver.instance.columnfamily.should eq(SPEC_COLUMN_FAMILY)
|
43
|
-
driver.instance.ttl.should eq(0)
|
44
|
-
end
|
45
|
-
|
46
|
-
it 'should configure ttl' do
|
47
|
-
ttl = 20
|
48
|
-
driver.configure(set_config_value(CONFIG, :ttl, ttl))
|
49
|
-
driver.instance.ttl.should eq(ttl)
|
50
|
-
end
|
51
|
-
|
52
|
-
describe 'exceptions' do
|
53
|
-
it 'should raise an exception if host is not configured' do
|
54
|
-
expect { driver.configure(CONFIG.gsub("host", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
55
|
-
end
|
56
|
-
|
57
|
-
it 'should raise an exception if port is not configured' do
|
58
|
-
expect { driver.configure(CONFIG.gsub("port", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
59
|
-
end
|
60
|
-
|
61
|
-
it 'should raise an exception if keyspace is not configured' do
|
62
|
-
expect { driver.configure(CONFIG.gsub("keyspace", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
63
|
-
end
|
64
|
-
|
65
|
-
it 'should raise an exception if columnfamily is not configured' do
|
66
|
-
expect { driver.configure(CONFIG.gsub("columnfamily", "invalid_config_name")) }.to raise_error Fluent::ConfigError
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
end # context configuring
|
71
|
-
|
72
|
-
context 'logging' do
|
73
|
-
|
74
|
-
it 'should start' do
|
75
|
-
driver.configure(CONFIG)
|
76
|
-
driver.instance.start
|
77
|
-
end
|
78
|
-
|
79
|
-
it 'should shutdown' do
|
80
|
-
driver.configure(CONFIG)
|
81
|
-
driver.instance.start
|
82
|
-
driver.instance.shutdown
|
83
|
-
end
|
84
|
-
|
85
|
-
it 'should format' do
|
86
|
-
driver.configure(CONFIG)
|
87
|
-
time = Time.now.to_i
|
88
|
-
record = {'tag' => 'test', 'time' => time, 'a' => 1}
|
89
|
-
|
90
|
-
driver.emit(record)
|
91
|
-
driver.expect_format(record.to_msgpack)
|
92
|
-
driver.run
|
93
|
-
end
|
94
|
-
|
95
|
-
context 'writing' do
|
96
|
-
context 'as json' do
|
97
|
-
|
98
|
-
describe 'pop no data keys' do
|
99
|
-
it 'should store json in last column' do
|
100
|
-
driver.configure(set_config_value(CONFIG, :pop_data_keys, false))
|
101
|
-
write(driver, SPEC_COLUMN_FAMILY, false)
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
describe 'pop some data keys' do
|
106
|
-
it 'should store json in last last column' do
|
107
|
-
driver.configure(set_config_value(CONFIG, :pop_data_keys, true))
|
108
|
-
write(driver, SPEC_COLUMN_FAMILY, false)
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
describe 'pop all data keys' do
|
113
|
-
it 'should store empty string in last column' do
|
114
|
-
driver.configure(CONFIG)
|
115
|
-
write(driver, SPEC_COLUMN_FAMILY, true)
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
end # context as json
|
120
|
-
|
121
|
-
context 'as columns' do # no need to test popping of keys b/c it makes no difference
|
122
|
-
|
123
|
-
it 'should write' do
|
124
|
-
config = set_config_value(CONFIG, :data_keys, DATA_KEYS + ',a')
|
125
|
-
config = set_config_value(CONFIG, :pop_data_keys, false)
|
126
|
-
driver.configure(config)
|
127
|
-
write(driver, SPEC_COLUMN_FAMILY, false)
|
128
|
-
end
|
129
|
-
|
130
|
-
end # context as columns
|
131
|
-
|
132
|
-
it 'should not locate event after ttl has expired' do
|
133
|
-
time = Time.now.to_i
|
134
|
-
tag = "ttl_test"
|
135
|
-
ttl = 1 # set ttl to 1 second
|
136
|
-
|
137
|
-
driver.configure(set_config_value(CONFIG, :ttl, ttl))
|
138
|
-
driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
|
139
|
-
driver.run
|
140
|
-
|
141
|
-
# verify record... should return in less than one sec if hitting
|
142
|
-
# cassandra running on localhost
|
143
|
-
events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
|
144
|
-
events.rows.should eq(1)
|
145
|
-
|
146
|
-
# now, sleep long enough for the event to be expired from cassandra
|
147
|
-
sleep(ttl + 1)
|
148
|
-
|
149
|
-
# re-query and verify that no events were returned
|
150
|
-
events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
|
151
|
-
events.rows.should eq(0)
|
152
|
-
end
|
153
|
-
|
154
|
-
end # context writing
|
155
|
-
|
156
|
-
end # context logging
|
157
|
-
|
158
|
-
end # CassandraOutput
|
data/spec/support/helpers.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
module Helpers
|
2
|
-
|
3
|
-
def write(driver, column_family_name, tag_and_time_only)
|
4
|
-
tag1 = "test1"
|
5
|
-
tag2 = "test2"
|
6
|
-
time1 = Time.now.to_i
|
7
|
-
time2 = Time.now.to_i + 2
|
8
|
-
|
9
|
-
record1 = {'tag' => tag1, 'time' => time1}
|
10
|
-
record2 = {'tag' => tag2, 'time' => time2}
|
11
|
-
|
12
|
-
unless tag_and_time_only
|
13
|
-
record1.merge!({'a' => 10, 'b' => 'Tesla'})
|
14
|
-
record2.merge!({'a' => 20, 'b' => 'Edison'})
|
15
|
-
end
|
16
|
-
|
17
|
-
# store both records in an array
|
18
|
-
records = [record1, record2]
|
19
|
-
|
20
|
-
driver.emit(records[0])
|
21
|
-
driver.emit(records[1])
|
22
|
-
driver.run # persists to cassandra
|
23
|
-
|
24
|
-
# query cassandra to verify data was correctly persisted
|
25
|
-
row_num = records.count # non-zero based index
|
26
|
-
events = driver.instance.session.execute("SELECT * FROM #{column_family_name}")
|
27
|
-
events.rows.should eq(records.count)
|
28
|
-
events.fetch do | event | # events should be sorted desc by tag, then time
|
29
|
-
row_num -= 1 # zero-based index
|
30
|
-
|
31
|
-
record = records[row_num]
|
32
|
-
db_hash = event.to_hash
|
33
|
-
|
34
|
-
# need to take in account that we've popped both tag and time
|
35
|
-
# from the payload data when we saved it
|
36
|
-
if driver.instance.pop_data_keys
|
37
|
-
db_hash['id'].should eq(record.delete('tag'))
|
38
|
-
db_hash['ts'].should eq(record.delete('time'))
|
39
|
-
else
|
40
|
-
db_hash['id'].should eq(record['tag'])
|
41
|
-
db_hash['ts'].should eq(record['time'])
|
42
|
-
end
|
43
|
-
|
44
|
-
if driver.instance.schema.keys.count == driver.instance.data_keys.count + 1 # store as json
|
45
|
-
if record.count > 0
|
46
|
-
db_hash['payload'].should eq(record.to_json)
|
47
|
-
else
|
48
|
-
db_hash['payload'].should eq('')
|
49
|
-
end
|
50
|
-
else
|
51
|
-
db_hash['payload'].should eq(record[record.keys[db_hash.keys.index('payload')]])
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|