fluent-plugin-cassandra-driver 0.0.17 → 0.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25ef90aa92ffd95954d6d08c03138a8f592754e0
-  data.tar.gz: 2fb7c13faae6a94e60d9dc237af0100cd89ba692
+  metadata.gz: 59f881900aec4466eb669a9223ae1d59f2665a72
+  data.tar.gz: dc9e4d39436b007ea6dcbb988715f4579b1f94cc
 SHA512:
-  metadata.gz: fbad4b4edb328675f0fcebcc6c2e56cb249fdffd3250b06ee48acfc6adbfc03976544538834caeb8a51d4e43acdb53faf0ef7255b6ed0defb2e5d88672f13fe0
-  data.tar.gz: 8c059b565f715bd75179f899951a1cc36e245fe62a034327ca46a0097eec30be88337c778ae03c8a4eaf30cab7d55f3d630adc89095ef46417174683084c72b5
+  metadata.gz: 0d2aa721973077d178571756c4d3af3c26267b7876001d06f204284bc594da0f3fa7841f030a0b5268206453be1273a5b77190bd115f624fdff559737dac6cd9
+  data.tar.gz: e569006e18921befbf58713238f5fdb3f04d5ca065083edf4e0e134191e1b4af263a01b2caf0a69a9b69b1831f3bf74db645a78a6e49edbd288184549afbd7be

data/README.md

@@ -10,17 +10,16 @@ and Cassandra 1.2 - 3.x
 via RubyGems
 
     fluent-gem install fluent-plugin-cassandra-driver
+    td-agent-gem install fluent-plugin-cassandra-driver
 
 # Quick Start
 
 ## Cassandra Configuration
-    # create keyspace (via CQL)
-      CREATE KEYSPACE \"metrics\" WITH strategy_class='org.apache.cassandra.locator.SimpleStrategy' AND strategy_options:replication_factor=1;
+    # Create keyspace (via CQL)
+      CREATE KEYSPACE metrics WITH strategy_class='org.apache.cassandra.locator.SimpleStrategy' AND strategy_options:replication_factor=1;
 
-    # create table (column family)
-      CREATE TABLE logs (id varchar, ts bigint, payload text, PRIMARY KEY (id, ts)) WITH CLUSTERING ORDER BY (ts DESC);
-
-    # NOTE: schema definition should match that specified in the Fluentd.conf configuration file (see below)
+    # Create table (column family)
+      CREATE TABLE logs (id varchar, timestamp timestamp, json text, PRIMARY KEY (id, timestamp)) WITH CLUSTERING ORDER BY (timestamp DESC);
 
 ## Fluentd.conf Configuration
     <match cassandra.**>
@@ -28,12 +27,24 @@ via RubyGems
       hosts 127.0.0.1            # comma delimited string of hosts
       keyspace metrics           # cassandra keyspace
       columnfamily logs          # cassandra column family
-      ttl 60                     # cassandra ttl *optional => default is 0*
-      schema                     # cassandra column family schema *hash where keys => column names and values => data types* for example: {:id => :string}
-      data_keys                  # comma delimited string of the fluentd hash's keys
-      pop_data_keys              # keep or pop key/values from the fluentd hash when storing it as json
+      ttl 60                     # cassandra ttl (optional, default is 0)
+      schema                     # cassandra column family schema (see example below)
+      pop_data_keys              # keep or pop key/values from the fluentd hash when storing it as json (optional, default is false)
+      json_column json           # column where store all remaining data from fluentd (optional)
     </match>
     
+### Schema example
+    # hash of hashes :column_damily_key => {:fluentd_record_key => :type_from_list}
+    '{:id => {:id => nil}, :timestamp => {:timestamp => :time}}'
+    
+Available mappings:
+* :integer
+* :string
+* :timeuuid
+* :time
+    
+All nil types will be recognized as string.
+    
 # Tests
 
 TODO

data/VERSION

@@ -1 +1 @@
-0.0.17
+0.0.18

data/fluent-plugin-cassandra-driver.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: fluent-plugin-cassandra-driver 0.0.17 ruby lib
+# stub: fluent-plugin-cassandra-driver 0.0.18 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-cassandra-driver".freeze
-  s.version = "0.0.17"
+  s.version = "0.0.18"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Yaroslav Lukyanov".freeze]
-  s.date = "2016-11-10"
+  s.date = "2016-11-12"
   s.description = "Fluent output plugin for Cassandra via Datastax Ruby Driver for Apache Cassandra".freeze
   s.email = "c_sharp@mail.ru".freeze
   s.extra_rdoc_files = [

data/lib/fluent/plugin/out_cassandra_driver.rb

@@ -8,15 +8,17 @@ module Fluent
 
     config_param :hosts, :string
     config_param :keyspace, :string
-    config_param :columnfamily, :string
+    config_param :column_family, :string
     config_param :ttl, :integer, :default => 0
     config_param :schema, :string
-    config_param :data_keys, :string
 
     # remove keys from the fluentd json event as they're processed
     # for individual columns?
     config_param :pop_data_keys, :bool, :default => true
 
+    # column to store all data keys as json
+    config_param :json_column, :string
+
     def session
       @session ||= get_session(self.hosts, self.keyspace)
     end
@@ -27,16 +29,16 @@ module Fluent
       # perform validations
       raise ConfigError, "'Hosts' is required by Cassandra output (ex: localhost, 127.0.0.1, ec2-54-242-141-252.compute-1.amazonaws.com" if self.hosts.nil?
       raise ConfigError, "'Keyspace' is required by Cassandra output (ex: FluentdLoggers)" if self.keyspace.nil?
-      raise ConfigError, "'ColumnFamily' is required by Cassandra output (ex: events)" if self.columnfamily.nil?
-      raise ConfigError, "'Schema' is required by Cassandra output (ex: id,ts,payload)" if self.schema.nil?
-      raise ConfigError, "'Schema' must contain at least two column names (ex: id,ts,payload)" if self.schema.split(',').count < 2
-      raise ConfigError, "'DataKeys' is required by Cassandra output (ex: tag,created_at,data)" if self.data_keys.nil?
+      raise ConfigError, "'ColumnFamily' is required by Cassandra output (ex: events)" if self.column_family.nil?
+      raise ConfigError, "'Schema' is required by Cassandra output" if self.schema.nil?
 
       # convert schema from string to hash
       # NOTE: ok to use eval b/c this isn't this isn't a user
       #       supplied string
       self.schema = eval(self.schema)
 
+      raise ConfigError, "'Schema' must contain at least one column" if self.schema.keys.length < 1
+
       # convert data keys from string to array
       self.data_keys = self.data_keys.split(',')
 
@@ -62,12 +64,12 @@ module Fluent
       chunk.msgpack_each { |record|
         $log.debug "Sending a new record to Cassandra: #{record.to_json}"
 
-        values = build_insert_values(self.schema.keys, self.data_keys, record, self.pop_data_keys)
+        values = build_insert_values(record)
 
-        cql = "INSERT INTO #{self.columnfamily} (#{self.schema.keys.join(',')}) VALUES (#{values.length.times.map { '?' }.join(',')}) USING TTL #{self.ttl}"
+        cql = "INSERT INTO #{self.column_family} (#{values.keys.join(',')}) VALUES (#{values.keys.map { |key| ":#{key}" }.join(',')}) USING TTL #{self.ttl}"
 
         $log.debug "CQL query: #{cql}"
-        $log.debug "Running with values: #{values.to_json}"
+        $log.debug "Running with arguments: #{values.to_json}"
 
         begin
           @session.execute(cql, arguments: values)
@@ -87,44 +89,31 @@ module Fluent
       cluster.connect(keyspace)
     end
 
-    def build_insert_values(schema_keys, data_keys, record, pop_data_keys)
-      values = data_keys.map.with_index do |key, index|
-        value = record[key]
+    def build_insert_values(record)
+      values = self.schema.map.with_index { |column_family_key, mapping|
+        record_key, type = mapping.first
+        value = record[record_key]
 
-        case self.schema[schema_keys[index]]
-          when :string
-            value = value.to_s
+        case type
           when :integer
             value = value.to_i
           when :timeuuid
             value = ::Cassandra::Uuid::Generator.new.at(Time.parse(value))
           when :time
             value = Time.parse(value)
+          when :string
           else
+            value = value.to_s
         end
 
-        value
-      end
-
-      data_keys.each { |key| record.delete(key) } if pop_data_keys
-
-      # if we have one more schema key than data keys,
-      # we can then infer that we should store the event
-      # as a string representation of the corresponding
-      # json object in the last schema column
-      if schema_keys.count == data_keys.count + 1
-        values << if record.count > 0
-                    "'#{record.to_json}'"
-                  else
-                    # by this point, the extra schema column has been
-                    # added to insert cql statement, so we must put
-                    # something in it
-                    # TODO: detect this scenario earlier and don't
-                    #       specify the column name/value at all
-                    #       when constructing the cql stmt
-                    "''"
-                  end
-      end
+        [column_family_key, value]
+      }.to_h
+
+      self.schema.each { |mapping| record.delete(mapping.first.first) } if self.pop_data_keys
+
+      # if we have one more data in record and json column
+      # then store all remaining data in that column
+      values[self.json_column] = record.to_json if self.json_column and record.length > 0
 
       values
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cassandra-driver
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
 platform: ruby
 authors:
 - Yaroslav Lukyanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-10 00:00:00.000000000 Z
+date: 2016-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd