fluent-plugin-cassandra-driver 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc1aec631bec8512c572f9c7b4eef5a2cfcb0a96
-  data.tar.gz: e64678e60780982ad114a504cf55e69735bdd1dc
+  metadata.gz: 7ac22674557433f0ea85c00863cee8ac7c59be11
+  data.tar.gz: 7e2f89d4571c7cc7801e6386458212ea3a2f6d33
 SHA512:
-  metadata.gz: 552915e4023e251d06f2d8b645a9e495a136a1d7953a34687467271c9c4e6670f5f66ad555abec6af7ba779a3162cc913afddba4894962447189477a7536d7f2
-  data.tar.gz: 22d47d1a6592b80938db0d192cc1a96aeeeb79c4c99f25be54a6646e2a4387bf80e39a78579abde35b38b2f81739f830f02ba6871fbb83c401e0f368d5918fbc
+  metadata.gz: 5c9d9fac0898971fac9967c015af167e0155dd0fd8cffe4fa12a1c1005c94c5e44c6000609a428fe45e1c9d5ebe9d8d64b666577830d8734a776b44040e1c0d6
+  data.tar.gz: 90128ed2a057a4618d1ce3b9104a8f635daaf7a4aeeb69e9b46d9e6dc6dcdeb763cb86238803b11ec557f0e8588270a088431ae6d008d52d8dc5e2ab533c3c56

data/Rakefile

@@ -2,6 +2,7 @@
 
 require 'rubygems'
 require 'bundler'
+
 begin
   Bundler.setup(:default, :development)
 rescue Bundler::BundlerError => e
@@ -9,9 +10,10 @@ rescue Bundler::BundlerError => e
   $stderr.puts 'Run `bundle install` to install missing gems'
   exit e.status_code
 end
-require 'rake'
 
+require 'rake'
 require 'jeweler'
+
 Jeweler::Tasks.new do |gem|
   # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
   gem.name = 'fluent-plugin-cassandra-driver'
@@ -23,14 +25,16 @@ Jeweler::Tasks.new do |gem|
   gem.authors = ['Yaroslav Lukyanov']
   # dependencies defined in Gemfile
 end
+
 Jeweler::RubygemsDotOrgTasks.new
 
 require 'rdoc/task'
+
 Rake::RDocTask.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
 
   rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "fluent-plugin-cassandra-cql #{version}"
+  rdoc.title = "fluent-plugin-cassandra-driver #{version}"
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/fluent-plugin-cassandra-driver.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: fluent-plugin-cassandra-driver 0.0.1 ruby lib
+# stub: fluent-plugin-cassandra-driver 0.0.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-cassandra-driver".freeze
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -28,10 +28,8 @@ Gem::Specification.new do |s|
     "VERSION",
     "fluent-plugin-cassandra-driver.gemspec",
     "lib/fluent/plugin/out_cassandra_driver.rb",
-    "spec/cassandra_cql_output_spec.rb",
     "spec/spec.opts",
-    "spec/spec_helper.rb",
-    "spec/support/helpers.rb"
+    "spec/spec_helper.rb"
   ]
   s.homepage = "https://github.com/CSharpRU/fluent-plugin-cassandra-driver".freeze
   s.licenses = ["BSD-3-Clause".freeze]

data/lib/fluent/plugin/out_cassandra_driver.rb

@@ -60,11 +60,17 @@ module Fluent
 
     def write(chunk)
       chunk.msgpack_each { |record|
+        $log.info "Sending a new record to Cassandra: #{record.to_json}"
+
         values = build_insert_values_string(self.schema.keys, self.data_keys, record, self.pop_data_keys)
 
         cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) VALUES (#{values}) USING TTL #{self.ttl}"
 
-        @session.execute(cql)
+        begin
+          @session.execute(cql)
+        rescue Exception => e
+          $log.error "Cannot send record to Cassandra: #{e.message}\nTrace: #{e.backtrace.to_s}"
+        end
       }
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cassandra-driver
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Yaroslav Lukyanov
@@ -126,10 +126,8 @@ files:
 - VERSION
 - fluent-plugin-cassandra-driver.gemspec
 - lib/fluent/plugin/out_cassandra_driver.rb
-- spec/cassandra_cql_output_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- spec/support/helpers.rb
 homepage: https://github.com/CSharpRU/fluent-plugin-cassandra-driver
 licenses:
 - BSD-3-Clause

data/spec/cassandra_cql_output_spec.rb

@@ -1,158 +0,0 @@
-require 'spec_helper'
-Fluent::Test.setup
-
-SPEC_COLUMN_FAMILY = "spec_events"
-DATA_KEYS = "tag,time"
-
-CONFIG = %[
-  host 127.0.0.1
-  port 9160
-  keyspace FluentdLoggers
-  columnfamily #{SPEC_COLUMN_FAMILY}
-  ttl 0
-  schema {:id => :string, :ts => :bigint, :payload => :string}
-  data_keys #{DATA_KEYS}
-  pop_data_keys true
-]
-
-describe Fluent::CassandraCqlOutput do
-  include Helpers
-
-  let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test') }
-
-  after(:each) do
-    d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraCqlOutput, 'test')
-    d.configure(CONFIG)
-    d.instance.session.execute("TRUNCATE #{SPEC_COLUMN_FAMILY}")
-  end
-
-  def set_config_value(config, config_name, value)
-    search_text = config.split("\n").map {|text| text if text.strip!.to_s.start_with? config_name.to_s}.compact![0]
-    config.gsub(search_text, "#{config_name} #{value}")
-  end
-
-  context 'configuring' do
-
-    it 'should be properly configured' do
-      driver.configure(CONFIG)
-      driver.tag.should eq('test')
-      driver.instance.host.should eq('127.0.0.1')
-      driver.instance.port.should eq(9160)
-      driver.instance.keyspace.should eq('FluentdLoggers')
-      driver.instance.columnfamily.should eq(SPEC_COLUMN_FAMILY)
-      driver.instance.ttl.should eq(0)
-    end
-
-    it 'should configure ttl' do
-      ttl = 20
-      driver.configure(set_config_value(CONFIG, :ttl, ttl))
-      driver.instance.ttl.should eq(ttl)
-    end
-
-    describe 'exceptions' do
-      it 'should raise an exception if host is not configured' do
-        expect { driver.configure(CONFIG.gsub("host", "invalid_config_name")) }.to raise_error Fluent::ConfigError
-      end
-
-      it 'should raise an exception if port is not configured' do
-        expect { driver.configure(CONFIG.gsub("port", "invalid_config_name")) }.to raise_error Fluent::ConfigError
-      end
-
-      it 'should raise an exception if keyspace is not configured' do
-        expect { driver.configure(CONFIG.gsub("keyspace", "invalid_config_name")) }.to raise_error Fluent::ConfigError
-      end
-
-      it 'should raise an exception if columnfamily is not configured' do
-        expect { driver.configure(CONFIG.gsub("columnfamily", "invalid_config_name")) }.to raise_error Fluent::ConfigError
-      end
-    end
-
-  end # context configuring
-
-  context 'logging' do
-
-    it 'should start' do
-      driver.configure(CONFIG)
-      driver.instance.start
-    end
-
-    it 'should shutdown' do
-      driver.configure(CONFIG)
-      driver.instance.start
-      driver.instance.shutdown
-    end
-
-    it 'should format' do
-      driver.configure(CONFIG)
-      time = Time.now.to_i
-      record = {'tag' => 'test', 'time' => time, 'a' => 1}
-
-      driver.emit(record)
-      driver.expect_format(record.to_msgpack)
-      driver.run
-    end
-
-    context 'writing' do
-      context 'as json' do
-
-        describe 'pop no data keys' do
-          it 'should store json in last column' do
-            driver.configure(set_config_value(CONFIG, :pop_data_keys, false))
-            write(driver, SPEC_COLUMN_FAMILY, false)
-          end
-        end
-
-        describe 'pop some data keys' do
-          it 'should store json in last last column' do
-            driver.configure(set_config_value(CONFIG, :pop_data_keys, true))
-            write(driver, SPEC_COLUMN_FAMILY, false)
-          end
-        end
-
-        describe 'pop all data keys' do
-          it 'should store empty string in last column' do
-            driver.configure(CONFIG)
-            write(driver, SPEC_COLUMN_FAMILY, true)
-          end
-        end
-
-      end # context as json
-
-      context 'as columns' do # no need to test popping of keys b/c it makes no difference
-
-        it 'should write' do
-          config = set_config_value(CONFIG, :data_keys, DATA_KEYS + ',a')
-          config = set_config_value(CONFIG, :pop_data_keys, false)
-          driver.configure(config)
-          write(driver, SPEC_COLUMN_FAMILY, false)
-        end
-
-      end # context as columns  
-
-      it 'should not locate event after ttl has expired' do
-        time = Time.now.to_i
-        tag = "ttl_test"
-        ttl = 1 # set ttl to 1 second
-
-        driver.configure(set_config_value(CONFIG, :ttl, ttl))
-        driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
-        driver.run
-
-        # verify record... should return in less than one sec if hitting
-        #                  cassandra running on localhost
-        events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
-        events.rows.should eq(1)
-
-        # now, sleep long enough for the event to be expired from cassandra
-        sleep(ttl + 1)
-
-        # re-query and verify that no events were returned
-        events = driver.instance.session.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
-        events.rows.should eq(0)
-      end
-
-    end # context writing
-
-  end # context logging
-
-end # CassandraOutput

data/spec/support/helpers.rb

@@ -1,56 +0,0 @@
-module Helpers
-  
-  def write(driver, column_family_name, tag_and_time_only)
-    tag1 = "test1"
-    tag2 = "test2"
-    time1 = Time.now.to_i
-    time2 = Time.now.to_i + 2
-
-    record1 = {'tag' => tag1, 'time' => time1}
-    record2 = {'tag' => tag2, 'time' => time2}
-
-    unless tag_and_time_only
-      record1.merge!({'a' => 10, 'b' => 'Tesla'})
-      record2.merge!({'a' => 20, 'b' => 'Edison'})
-    end
-
-    # store both records in an array
-    records = [record1, record2]
-
-    driver.emit(records[0])
-    driver.emit(records[1])
-    driver.run # persists to cassandra
-
-    # query cassandra to verify data was correctly persisted
-    row_num = records.count # non-zero based index
-    events = driver.instance.session.execute("SELECT * FROM #{column_family_name}")
-    events.rows.should eq(records.count)
-    events.fetch do | event | # events should be sorted desc by tag, then time
-      row_num -= 1 # zero-based index
-
-      record = records[row_num]
-      db_hash = event.to_hash
-
-      # need to take in account that we've popped both tag and time
-      # from the payload data when we saved it
-      if driver.instance.pop_data_keys
-        db_hash['id'].should eq(record.delete('tag'))
-        db_hash['ts'].should eq(record.delete('time'))
-      else
-        db_hash['id'].should eq(record['tag'])
-        db_hash['ts'].should eq(record['time'])
-      end
-
-      if driver.instance.schema.keys.count == driver.instance.data_keys.count + 1 # store as json
-        if record.count > 0
-          db_hash['payload'].should eq(record.to_json)
-        else
-          db_hash['payload'].should eq('')
-        end
-      else
-        db_hash['payload'].should eq(record[record.keys[db_hash.keys.index('payload')]])
-      end
-    end
-  end
-
-end