fluent-plugin-cassandra-driver 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dc1aec631bec8512c572f9c7b4eef5a2cfcb0a96
4
- data.tar.gz: e64678e60780982ad114a504cf55e69735bdd1dc
3
+ metadata.gz: 7ac22674557433f0ea85c00863cee8ac7c59be11
4
+ data.tar.gz: 7e2f89d4571c7cc7801e6386458212ea3a2f6d33
5
5
  SHA512:
6
- metadata.gz: 552915e4023e251d06f2d8b645a9e495a136a1d7953a34687467271c9c4e6670f5f66ad555abec6af7ba779a3162cc913afddba4894962447189477a7536d7f2
7
- data.tar.gz: 22d47d1a6592b80938db0d192cc1a96aeeeb79c4c99f25be54a6646e2a4387bf80e39a78579abde35b38b2f81739f830f02ba6871fbb83c401e0f368d5918fbc
6
+ metadata.gz: 5c9d9fac0898971fac9967c015af167e0155dd0fd8cffe4fa12a1c1005c94c5e44c6000609a428fe45e1c9d5ebe9d8d64b666577830d8734a776b44040e1c0d6
7
+ data.tar.gz: 90128ed2a057a4618d1ce3b9104a8f635daaf7a4aeeb69e9b46d9e6dc6dcdeb763cb86238803b11ec557f0e8588270a088431ae6d008d52d8dc5e2ab533c3c56
data/Rakefile CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'rubygems'
4
4
  require 'bundler'
5
+
5
6
  begin
6
7
  Bundler.setup(:default, :development)
7
8
  rescue Bundler::BundlerError => e
@@ -9,9 +10,10 @@ rescue Bundler::BundlerError => e
9
10
  $stderr.puts 'Run `bundle install` to install missing gems'
10
11
  exit e.status_code
11
12
  end
12
- require 'rake'
13
13
 
14
+ require 'rake'
14
15
  require 'jeweler'
16
+
15
17
  Jeweler::Tasks.new do |gem|
16
18
  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
17
19
  gem.name = 'fluent-plugin-cassandra-driver'
@@ -23,14 +25,16 @@ Jeweler::Tasks.new do |gem|
23
25
  gem.authors = ['Yaroslav Lukyanov']
24
26
  # dependencies defined in Gemfile
25
27
  end
28
+
26
29
  Jeweler::RubygemsDotOrgTasks.new
27
30
 
28
31
  require 'rdoc/task'
32
+
29
33
  Rake::RDocTask.new do |rdoc|
30
34
  version = File.exist?('VERSION') ? File.read('VERSION') : ""
31
35
 
32
36
  rdoc.rdoc_dir = 'rdoc'
33
- rdoc.title = "fluent-plugin-cassandra-cql #{version}"
37
+ rdoc.title = "fluent-plugin-cassandra-driver #{version}"
34
38
  rdoc.rdoc_files.include('README*')
35
39
  rdoc.rdoc_files.include('lib/**/*.rb')
36
40
  end
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.1
1
+ 0.0.2
@@ -2,11 +2,11 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: fluent-plugin-cassandra-driver 0.0.1 ruby lib
5
+ # stub: fluent-plugin-cassandra-driver 0.0.2 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "fluent-plugin-cassandra-driver".freeze
9
- s.version = "0.0.1"
9
+ s.version = "0.0.2"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib".freeze]
@@ -28,10 +28,8 @@ Gem::Specification.new do |s|
28
28
  "VERSION",
29
29
  "fluent-plugin-cassandra-driver.gemspec",
30
30
  "lib/fluent/plugin/out_cassandra_driver.rb",
31
- "spec/cassandra_cql_output_spec.rb",
32
31
  "spec/spec.opts",
33
- "spec/spec_helper.rb",
34
- "spec/support/helpers.rb"
32
+ "spec/spec_helper.rb"
35
33
  ]
36
34
  s.homepage = "https://github.com/CSharpRU/fluent-plugin-cassandra-driver".freeze
37
35
  s.licenses = ["BSD-3-Clause".freeze]
@@ -60,11 +60,17 @@ module Fluent
60
60
 
61
61
  def write(chunk)
62
62
  chunk.msgpack_each { |record|
63
+ $log.info "Sending a new record to Cassandra: #{record.to_json}"
64
+
63
65
  values = build_insert_values_string(self.schema.keys, self.data_keys, record, self.pop_data_keys)
64
66
 
65
67
  cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) VALUES (#{values}) USING TTL #{self.ttl}"
66
68
 
67
- @session.execute(cql)
69
+ begin
70
+ @session.execute(cql)
71
+ rescue Exception => e
72
+ $log.error "Cannot send record to Cassandra: #{e.message}\nTrace: #{e.backtrace.to_s}"
73
+ end
68
74
  }
69
75
  end
70
76
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cassandra-driver
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yaroslav Lukyanov
@@ -126,10 +126,8 @@ files:
126
126
  - VERSION
127
127
  - fluent-plugin-cassandra-driver.gemspec
128
128
  - lib/fluent/plugin/out_cassandra_driver.rb
129
- - spec/cassandra_cql_output_spec.rb
130
129
  - spec/spec.opts
131
130
  - spec/spec_helper.rb
132
- - spec/support/helpers.rb
133
131
  homepage: https://github.com/CSharpRU/fluent-plugin-cassandra-driver
134
132
  licenses:
135
133
  - BSD-3-Clause
@@ -1,158 +0,0 @@
1
- require 'spec_helper'
2
- Fluent::Test.setup
3
-
4
- SPEC_COLUMN_FAMILY = "spec_events"
5
- DATA_KEYS = "tag,time"
6
-
7
- CONFIG = %[
8
- host 127.0.0.1
9
- port 9160
10
- keyspace FluentdLoggers
11
- columnfamily #{SPEC_COLUMN_FAMILY}
12
- ttl 0
13
- schema {:id => :string, :ts => :bigint, :payload => :string}
14
- data_keys #{DATA_KEYS}
15
- pop_data_keys true
16
- ]
17
-
18
- describe Fluent::CassandraCqlOutput do
19
- include Helpers
20
-
21
- let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test') }
22
-
23
- after(:each) do
24
- d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test')
25
- d.configure(CONFIG)
26
- d.instance.session.execute("TRUNCATE #{SPEC_COLUMN_FAMILY}")
27
- end
28
-
29
- def set_config_value(config, config_name, value)
30
- search_text = config.split("\n").map {|text| text if text.strip!.to_s.start_with? config_name.to_s}.compact![0]
31
- config.gsub(search_text, "#{config_name} #{value}")
32
- end
33
-
34
- context 'configuring' do
35
-
36
- it 'should be properly configured' do
37
- driver.configure(CONFIG)
38
- driver.tag.should eq('test')
39
- driver.instance.host.should eq('127.0.0.1')
40
- driver.instance.port.should eq(9160)
41
- driver.instance.keyspace.should eq('FluentdLoggers')
42
- driver.instance.columnfamily.should eq(SPEC_COLUMN_FAMILY)
43
- driver.instance.ttl.should eq(0)
44
- end
45
-
46
- it 'should configure ttl' do
47
- ttl = 20
48
- driver.configure(set_config_value(CONFIG, :ttl, ttl))
49
- driver.instance.ttl.should eq(ttl)
50
- end
51
-
52
- describe 'exceptions' do
53
- it 'should raise an exception if host is not configured' do
54
- expect { driver.configure(CONFIG.gsub("host", "invalid_config_name")) }.to raise_error Fluent::ConfigError
55
- end
56
-
57
- it 'should raise an exception if port is not configured' do
58
- expect { driver.configure(CONFIG.gsub("port", "invalid_config_name")) }.to raise_error Fluent::ConfigError
59
- end
60
-
61
- it 'should raise an exception if keyspace is not configured' do
62
- expect { driver.configure(CONFIG.gsub("keyspace", "invalid_config_name")) }.to raise_error Fluent::ConfigError
63
- end
64
-
65
- it 'should raise an exception if columnfamily is not configured' do
66
- expect { driver.configure(CONFIG.gsub("columnfamily", "invalid_config_name")) }.to raise_error Fluent::ConfigError
67
- end
68
- end
69
-
70
- end # context configuring
71
-
72
- context 'logging' do
73
-
74
- it 'should start' do
75
- driver.configure(CONFIG)
76
- driver.instance.start
77
- end
78
-
79
- it 'should shutdown' do
80
- driver.configure(CONFIG)
81
- driver.instance.start
82
- driver.instance.shutdown
83
- end
84
-
85
- it 'should format' do
86
- driver.configure(CONFIG)
87
- time = Time.now.to_i
88
- record = {'tag' => 'test', 'time' => time, 'a' => 1}
89
-
90
- driver.emit(record)
91
- driver.expect_format(record.to_msgpack)
92
- driver.run
93
- end
94
-
95
- context 'writing' do
96
- context 'as json' do
97
-
98
- describe 'pop no data keys' do
99
- it 'should store json in last column' do
100
- driver.configure(set_config_value(CONFIG, :pop_data_keys, false))
101
- write(driver, SPEC_COLUMN_FAMILY, false)
102
- end
103
- end
104
-
105
- describe 'pop some data keys' do
106
- it 'should store json in last last column' do
107
- driver.configure(set_config_value(CONFIG, :pop_data_keys, true))
108
- write(driver, SPEC_COLUMN_FAMILY, false)
109
- end
110
- end
111
-
112
- describe 'pop all data keys' do
113
- it 'should store empty string in last column' do
114
- driver.configure(CONFIG)
115
- write(driver, SPEC_COLUMN_FAMILY, true)
116
- end
117
- end
118
-
119
- end # context as json
120
-
121
- context 'as columns' do # no need to test popping of keys b/c it makes no difference
122
-
123
- it 'should write' do
124
- config = set_config_value(CONFIG, :data_keys, DATA_KEYS + ',a')
125
- config = set_config_value(CONFIG, :pop_data_keys, false)
126
- driver.configure(config)
127
- write(driver, SPEC_COLUMN_FAMILY, false)
128
- end
129
-
130
- end # context as columns
131
-
132
- it 'should not locate event after ttl has expired' do
133
- time = Time.now.to_i
134
- tag = "ttl_test"
135
- ttl = 1 # set ttl to 1 second
136
-
137
- driver.configure(set_config_value(CONFIG, :ttl, ttl))
138
- driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
139
- driver.run
140
-
141
- # verify record... should return in less than one sec if hitting
142
- # cassandra running on localhost
143
- events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
144
- events.rows.should eq(1)
145
-
146
- # now, sleep long enough for the event to be expired from cassandra
147
- sleep(ttl + 1)
148
-
149
- # re-query and verify that no events were returned
150
- events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
151
- events.rows.should eq(0)
152
- end
153
-
154
- end # context writing
155
-
156
- end # context logging
157
-
158
- end # CassandraOutput
@@ -1,56 +0,0 @@
1
- module Helpers
2
-
3
- def write(driver, column_family_name, tag_and_time_only)
4
- tag1 = "test1"
5
- tag2 = "test2"
6
- time1 = Time.now.to_i
7
- time2 = Time.now.to_i + 2
8
-
9
- record1 = {'tag' => tag1, 'time' => time1}
10
- record2 = {'tag' => tag2, 'time' => time2}
11
-
12
- unless tag_and_time_only
13
- record1.merge!({'a' => 10, 'b' => 'Tesla'})
14
- record2.merge!({'a' => 20, 'b' => 'Edison'})
15
- end
16
-
17
- # store both records in an array
18
- records = [record1, record2]
19
-
20
- driver.emit(records[0])
21
- driver.emit(records[1])
22
- driver.run # persists to cassandra
23
-
24
- # query cassandra to verify data was correctly persisted
25
- row_num = records.count # non-zero based index
26
- events = driver.instance.session.execute("SELECT * FROM #{column_family_name}")
27
- events.rows.should eq(records.count)
28
- events.fetch do | event | # events should be sorted desc by tag, then time
29
- row_num -= 1 # zero-based index
30
-
31
- record = records[row_num]
32
- db_hash = event.to_hash
33
-
34
- # need to take in account that we've popped both tag and time
35
- # from the payload data when we saved it
36
- if driver.instance.pop_data_keys
37
- db_hash['id'].should eq(record.delete('tag'))
38
- db_hash['ts'].should eq(record.delete('time'))
39
- else
40
- db_hash['id'].should eq(record['tag'])
41
- db_hash['ts'].should eq(record['time'])
42
- end
43
-
44
- if driver.instance.schema.keys.count == driver.instance.data_keys.count + 1 # store as json
45
- if record.count > 0
46
- db_hash['payload'].should eq(record.to_json)
47
- else
48
- db_hash['payload'].should eq('')
49
- end
50
- else
51
- db_hash['payload'].should eq(record[record.keys[db_hash.keys.index('payload')]])
52
- end
53
- end
54
- end
55
-
56
- end