fluent-plugin-cassandra-cql 0.0.1 → 0.0.2

data/README.md

@@ -2,12 +2,11 @@
 
 Cassandra output plugin for Fluentd.
 
-Implemented using the cassandra-cql gem and targets CQL 3.0.0
+Implemented using the cassandra-cql gem and targets [CQL 3.0.0](http://www.datastax.com/docs/1.1/references/cql/index)
 and Cassandra 1.1.x
 
 # Raison d'être
-Currently, there's another Fluentd Cassandra plugin [see
-here](https://github.com/tomitakazutaka/fluent-plugin-cassandra)
+Currently, there's another [Fluentd Cassandra plugin](https://github.com/tomitakazutaka/fluent-plugin-cassandra)
 
 It's implemented via the Twitter Cassandra gem, which:
 
@@ -30,7 +29,8 @@ via RubyGems
 
     # create table (column family)
       CREATE TABLE events (id varchar, ts bigint, payload text, PRIMARY KEY (id, ts)) WITH CLUSTERING ORDER BY (ts DESC);
-      
+
+    # NOTE: schema definition should match that specified in the Fluentd.conf configuration file
 
 ## Fluentd.conf Configuration
     <match cassandra.**>
@@ -38,17 +38,21 @@ via RubyGems
       host 127.0.0.1             # cassandra hostname.
       port 9160                  # cassandra thrft port.
       keyspace FluentdLoggers    # cassandra keyspace
-      columnfamily events        # cassandra column family
+      columnfamily spec_events   # cassandra column family
       ttl 60                     # cassandra ttl *optional => default is 0*
+      schema                     # cassandra column family schema *hash where keys => column names and values => data types*
+      data_keys                  # comma delimited string of the fluentd hash's keys
+      pop_data_keys              # keep or pop key/values from the fluentd hash when storing it as json
     </match>
 
 # Tests
 
 rake rspec
 
-NOTE: requires that cassandra be installed on the machine running the tests
+    NOTE: requires that cassandra be installed on the machine running the
+          test as well as a keyspace named "FluentdLoggers" and a column family
+          named "spec_events"
 
 # TODOs
     1) make host and port configurable for tests
-    2) make schema definition configurable
-    3) add rake task to generate keyspace and columnfamily
+    2) add rake task to generate keyspace and columnfamily

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/fluent-plugin-cassandra-cql.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-cassandra-cql"
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["obie quelland"]
-  s.date = "2012-11-07"
+  s.date = "2012-11-11"
   s.description = "Fluent output plugin for Cassandra via CQL version 3.0.0"
   s.email = "quelland@gmail.com"
   s.extra_rdoc_files = [
@@ -28,7 +28,8 @@ Gem::Specification.new do |s|
     "lib/fluent/plugin/out_cassandra.rb",
     "spec/cassandra_output_spec.rb",
     "spec/spec.opts",
-    "spec/spec_helper.rb"
+    "spec/spec_helper.rb",
+    "spec/support/helpers.rb"
   ]
   s.homepage = "http://github.com/obieq/fluent-plugin-cassandra-cql"
   s.licenses = ["MIT"]

data/lib/fluent/plugin/out_cassandra.rb

@@ -6,33 +6,42 @@ module Fluent
 
   class CassandraOutput < BufferedOutput
     Fluent::Plugin.register_output('cassandra', self)
-    include SetTimeKeyMixin
-    include SetTagKeyMixin
 
-    config_param :host,         :string
-    config_param :port,         :integer
-    config_param :keyspace,     :string
-    config_param :columnfamily, :string
-    config_param :ttl,          :integer, :default => 0
+    config_param :host,          :string
+    config_param :port,          :integer
+    config_param :keyspace,      :string
+    config_param :columnfamily,  :string
+    config_param :ttl,           :integer, :default => 0
+    config_param :schema,        :string
+    config_param :data_keys,     :string
+
+    # remove keys from the fluentd json event as they're processed
+    # for individual columns?
+    config_param :pop_data_keys, :bool, :default => true
 
     def connection
-      @connection ||= get_connection
+      @connection ||= get_connection(self.host, self.port, self.keyspace)
     end
 
-    #config_set_default :include_time_key, true
-    #config_set_default :include_tag_key, true
-    #config_set_default :time_format, "%Y%m%d%H%M%S"
-
     def configure(conf)
       super
 
-      raise ConfigError, "'Host' is required by Cassandra output (ex: localhost, 127.0.0.1, ec2-54-242-141-252.compute-1.amazonaws.com" unless self.keyspace = conf['keyspace']
-      raise ConfigError, "'Port' is required by Cassandra output (ex: 9160)" unless self.columnfamily = conf['columnfamily']
-      raise ConfigError, "'Keyspace' is required by Cassandra output (ex: FluentdLoggers)" unless self.keyspace = conf['keyspace']
-      raise ConfigError, "'ColumnFamily' is required by Cassandra output (ex: events)" unless self.columnfamily = conf['columnfamily']
-
-      #@host = conf.has_key?('host') ? conf['host'] : 'localhost'
-      #@port = conf.has_key?('port') ? conf['port'] : 9160
+      # perform validations
+      raise ConfigError, "'Host' is required by Cassandra output (ex: localhost, 127.0.0.1, ec2-54-242-141-252.compute-1.amazonaws.com" if self.host.nil?
+      raise ConfigError, "'Port' is required by Cassandra output (ex: 9160)" if self.port.nil?
+      raise ConfigError, "'Keyspace' is required by Cassandra output (ex: FluentdLoggers)" if self.keyspace.nil?
+      raise ConfigError, "'ColumnFamily' is required by Cassandra output (ex: events)" if self.columnfamily.nil?
+      raise ConfigError, "'Schema' is required by Cassandra output (ex: id,ts,payload)" if self.schema.nil?
+      raise ConfigError, "'Schema' must contain at least two column names (ex: id,ts,payload)" if self.schema.split(',').count < 2
+      raise ConfigError, "'DataKeys' is required by Cassandra output (ex: tag,created_at,data)" if self.data_keys.nil?
+
+      # convert schema from string to hash
+      # NOTE: ok to use eval b/c this isn't this isn't a user
+      #       supplied string
+      self.schema = eval(self.schema)
+
+      # convert data keys from string to array
+      self.data_keys = self.data_keys.split(',')
     end
 
     def start
@@ -50,17 +59,49 @@ module Fluent
 
     def write(chunk)
       chunk.msgpack_each  { |record|
-        @connection.execute("INSERT INTO #{self.columnfamily} (id, ts, payload) " +
-                            "VALUES ('#{record['tag']}', #{record['time']}, '#{record.to_json}') " +
-                            "USING TTL #{self.ttl}")
+        values = build_insert_values_string(self.schema.keys, self.data_keys, record, self.pop_data_keys)
+        cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) " +
+                            "VALUES (#{values}) " +
+                            "USING TTL #{self.ttl}"
+        @connection.execute(cql)
       }
     end
 
     private
 
-    def get_connection
-      connection_string = "#{self.host}:#{self.port}"
-      ::CassandraCQL::Database.new(connection_string, {:keyspace => "\"#{self.keyspace}\"", :cql_version => "3.0.0"})
+    def get_connection(host, port, keyspace)
+      connection_string = "#{host}:#{port}"
+      ::CassandraCQL::Database.new(connection_string, {:keyspace => "\"#{keyspace}\"", :cql_version => "3.0.0"})
+    end
+
+    def build_insert_values_string(schema_keys, data_keys, record, pop_data_keys)
+      values = data_keys.map.with_index do |key, index|
+        if pop_data_keys
+          schema[schema_keys[index]] == :string ? "'#{record.delete(key)}'" : record.delete(key)
+        else
+          schema[schema_keys[index]] == :string ? "'#{record[key]}'" : record[key]
+        end
+      end
+
+      # if we have one more schema key than data keys,
+      # we can then infer that we should store the event
+      # as a string representation of the corresponding
+      # json object in the last schema column
+      if schema_keys.count == data_keys.count + 1
+        values << if record.count > 0
+                    "'#{record.to_json}'"
+                  else
+                    # by this point, the extra schema column has been
+                    # added to insert cql statement, so we must put
+                    # something in it
+                    # TODO: detect this scenario earlier and don't
+                    #       specify the column name/value at all
+                    #       when constructing the cql stmt
+                    "''"
+                  end
+      end
+
+      return values.join(',')
     end
 
   end

data/spec/cassandra_output_spec.rb

@@ -1,24 +1,34 @@
 require 'spec_helper'
 Fluent::Test.setup
 
+SPEC_COLUMN_FAMILY = "spec_events"
+DATA_KEYS = "tag,time"
+
 CONFIG = %[
   host 127.0.0.1
   port 9160
   keyspace FluentdLoggers
-  columnfamily events
+  columnfamily #{SPEC_COLUMN_FAMILY}
+  ttl 0
+  schema {:id => :string, :ts => :bigint, :payload => :string}
+  data_keys #{DATA_KEYS}
+  pop_data_keys true
 ]
 
 describe Fluent::CassandraOutput do
+  include Helpers
+
   let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraOutput, 'test') }
 
   after(:each) do
     d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::CassandraOutput, 'test')
     d.configure(CONFIG)
-    d.instance.connection.execute("TRUNCATE events")
+    d.instance.connection.execute("TRUNCATE #{SPEC_COLUMN_FAMILY}")
   end
 
-  def add_ttl_to_config(ttl)
-    return CONFIG + %[  ttl #{ttl}\n]
+  def set_config_value(config, config_name, value)
+    search_text = config.split("\n").map {|text| text if text.strip!.to_s.start_with? config_name.to_s}.compact![0]
+    config.gsub(search_text, "#{config_name} #{value}")
   end
 
   context 'configuring' do
@@ -29,18 +39,17 @@ describe Fluent::CassandraOutput do
       driver.instance.host.should eq('127.0.0.1')
       driver.instance.port.should eq(9160)
       driver.instance.keyspace.should eq('FluentdLoggers')
-      driver.instance.columnfamily.should eq('events')
+      driver.instance.columnfamily.should eq(SPEC_COLUMN_FAMILY)
       driver.instance.ttl.should eq(0)
     end
 
     it 'should configure ttl' do
       ttl = 20
-      driver.configure(add_ttl_to_config(ttl))
+      driver.configure(set_config_value(CONFIG, :ttl, ttl))
       driver.instance.ttl.should eq(ttl)
     end
 
     describe 'exceptions' do
-
       it 'should raise an exception if host is not configured' do
         expect { driver.configure(CONFIG.gsub("host", "invalid_config_name")) }.to raise_error Fluent::ConfigError
       end
@@ -56,12 +65,11 @@ describe Fluent::CassandraOutput do
       it 'should raise an exception if columnfamily is not configured' do
         expect { driver.configure(CONFIG.gsub("columnfamily", "invalid_config_name")) }.to raise_error Fluent::ConfigError
       end
-
     end
 
-  end
+  end # context configuring
 
-  context 'fluentd logging' do
+  context 'logging' do
 
     it 'should start' do
       driver.configure(CONFIG)
@@ -84,55 +92,67 @@ describe Fluent::CassandraOutput do
       driver.run
     end
 
-    it 'should write' do
-      driver.configure(CONFIG)
-      tag1 = "test1"
-      tag2 = "test2"
-      time1 = Time.now.to_i
-      time2 = Time.now.to_i + 2
-      record1 = {'tag' => tag1, 'time' => time1, 'a' => 10, 'b' => 'Tesla'}
-      record2 = {'tag' => tag2, 'time' => time2, 'a' => 20, 'b' => 'Edison'}
-      records = [record1, record2]
-
-      driver.emit(records[0])
-      driver.emit(records[1])
-      driver.run # persists to cassandra
-
-      # query cassandra to verify data was correctly persisted
-      row_num = records.count # non-zero based index
-      events = driver.instance.connection.execute("SELECT * FROM events")
-      events.rows.should eq(records.count)
-      events.fetch do | event | # events should be sorted desc by tag, then time
-        row_num -= 1 # zero-based index
-        hash = event.to_hash
-        hash['id'].should eq(records[row_num]['tag'])
-        hash['ts'].should eq(records[row_num]['time'])
-        hash['payload'].should eq(records[row_num].to_json)
+    context 'writing' do
+      context 'as json' do
+
+        describe 'pop no data keys' do
+          it 'should store json in last column' do
+            driver.configure(set_config_value(CONFIG, :pop_data_keys, false))
+            write(driver, SPEC_COLUMN_FAMILY, false)
+          end
+        end
+
+        describe 'pop some data keys' do
+          it 'should store json in last last column' do
+            driver.configure(set_config_value(CONFIG, :pop_data_keys, true))
+            write(driver, SPEC_COLUMN_FAMILY, false)
+          end
+        end
+
+        describe 'pop all data keys' do
+          it 'should store empty string in last column' do
+            driver.configure(CONFIG)
+            write(driver, SPEC_COLUMN_FAMILY, true)
+          end
+        end
+
+      end # context as json
+
+      context 'as columns' do # no need to test popping of keys b/c it makes no difference
+
+        it 'should write' do
+          config = set_config_value(CONFIG, :data_keys, DATA_KEYS + ',a')
+          config = set_config_value(CONFIG, :pop_data_keys, false)
+          driver.configure(config)
+          write(driver, SPEC_COLUMN_FAMILY, false)
+        end
+
+      end # context as columns  
+
+      it 'should not locate event after ttl has expired' do
+        time = Time.now.to_i
+        tag = "ttl_test"
+        ttl = 1 # set ttl to 1 second
+
+        driver.configure(set_config_value(CONFIG, :ttl, ttl))
+        driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
+        driver.run
+
+        # verify record... should return in less than one sec if hitting
+        #                  cassandra running on localhost
+        events = driver.instance.connection.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
+        events.rows.should eq(1)
+
+        # now, sleep long enough for the event to be expired from cassandra
+        sleep(ttl + 1)
+
+        # re-query and verify that no events were returned
+        events = driver.instance.connection.execute("SELECT * FROM #{SPEC_COLUMN_FAMILY} where ts = #{time}")
+        events.rows.should eq(0)
       end
-    end
 
-    it 'should not locate event after ttl has expired' do
-      time = Time.now.to_i
-      tag = "ttl_test"
-      ttl = 1 # set ttl to 1 second
+    end # context writing
 
-      driver.configure(add_ttl_to_config(ttl))
-      driver.emit({'tag' => tag, 'time' => time, 'a' => 1})
-      driver.run
-
-      # verify record... should return in less than one sec if hitting
-      #                  cassandra running on localhost
-      events = driver.instance.connection.execute("SELECT * FROM events where ts = #{time}")
-      events.rows.should eq(1)
-
-      # now, sleep long enough for the event to be expired from cassandra
-      sleep(ttl)
-
-      # re-query and verify that no events were returned
-      events = driver.instance.connection.execute("SELECT * FROM events where ts = #{time}")
-      events.rows.should eq(0)
-    end
-
-  end
+  end # context logging
 
 end # CassandraOutput

data/spec/spec_helper.rb

@@ -6,4 +6,4 @@ require 'fluent/test'
 Dir["./lib/**/*.rb"].each {|f| require f}
 
 # require the shared example files
-#Dir["./spec/support/**/*.rb"].each {|f| require f}
+Dir["./spec/support/**/*.rb"].each {|f| require f}

data/spec/support/helpers.rb

@@ -0,0 +1,56 @@
+module Helpers
+  
+  def write(driver, column_family_name, tag_and_time_only)
+    tag1 = "test1"
+    tag2 = "test2"
+    time1 = Time.now.to_i
+    time2 = Time.now.to_i + 2
+
+    record1 = {'tag' => tag1, 'time' => time1}
+    record2 = {'tag' => tag2, 'time' => time2}
+
+    unless tag_and_time_only
+      record1.merge!({'a' => 10, 'b' => 'Tesla'})
+      record2.merge!({'a' => 20, 'b' => 'Edison'})
+    end
+
+    # store both records in an array
+    records = [record1, record2]
+
+    driver.emit(records[0])
+    driver.emit(records[1])
+    driver.run # persists to cassandra
+
+    # query cassandra to verify data was correctly persisted
+    row_num = records.count # non-zero based index
+    events = driver.instance.connection.execute("SELECT * FROM #{column_family_name}")
+    events.rows.should eq(records.count)
+    events.fetch do | event | # events should be sorted desc by tag, then time
+      row_num -= 1 # zero-based index
+
+      record = records[row_num]
+      db_hash = event.to_hash
+
+      # need to take in account that we've popped both tag and time
+      # from the payload data when we saved it
+      if driver.instance.pop_data_keys
+        db_hash['id'].should eq(record.delete('tag'))
+        db_hash['ts'].should eq(record.delete('time'))
+      else
+        db_hash['id'].should eq(record['tag'])
+        db_hash['ts'].should eq(record['time'])
+      end
+
+      if driver.instance.schema.keys.count == driver.instance.data_keys.count + 1 # store as json
+        if record.count > 0
+          db_hash['payload'].should eq(record.to_json)
+        else
+          db_hash['payload'].should eq('')
+        end
+      else
+        db_hash['payload'].should eq(record[record.keys[db_hash.keys.index('payload')]])
+      end
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cassandra-cql
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-07 00:00:00.000000000 Z
+date: 2012-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -143,6 +143,7 @@ files:
 - spec/cassandra_output_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/support/helpers.rb
 homepage: http://github.com/obieq/fluent-plugin-cassandra-cql
 licenses:
 - MIT
@@ -158,7 +159,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -709806578343991657
+      hash: -4467205590141374709
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: