fluent-plugin-cassandra-json 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 124d011ac7e9cef45d13531d991010f90583bc7d121ec8dcfa0ef48997582aa0
-  data.tar.gz: 46ea571783aee03416d936d992673869fa117e4fbefc14616c636560f0cd536e
+  metadata.gz: de8e8cdfc8a0ee72d199f0e3529b59ea56eb087afd31e6f88cf50c48d8ac426a
+  data.tar.gz: 8f41c9632542d10f41fccd3998273a6baec5f911ca70e788bac2e84a71181b59
 SHA512:
-  metadata.gz: 6a138f83bf458a88754680f515a28631984b5f44b2fdea011a270a4c1a57df4c02fdff4f0b6782b4a5c669b4071bd11abcf6996c7edd515ca00ef586d00bc4bc
-  data.tar.gz: bd7aa50cbf401906f39d4b0bdfad226d59022cdb122c87211a270998bead1e802df59ec7ccb8d35c5aa4779b7b8a4490b096bea1821f28415bbf3d29ad0266df
+  metadata.gz: 3533891f9b9e23ed8d5a06c2a9d0ec9f6f8efe6eb7b15bc0dd5ded34536efd85546a36a830c01b41bf755e20d8efd59c8bb28e85642a231bf7775c2171907741
+  data.tar.gz: 760a800663413e5de4d4447a93d36b49acba6a261eec417108bee24ff13267ac385da542fafc85b706c1f5c9cf4360089c81814213c2491b175a3b3e97e9d687

data/README.md

@@ -87,6 +87,14 @@ Use TTL option on INSERT
 
 Specify whether this statement can be retried safely on timeout
 
+Default value: `true`.
+
+### default_unset (bool) (optional)
+
+Specify whether column not defined in the JSON is set to null or is ignored (If false, column not defined in the JSON is set to null. It is cassandra default)
+
+Default value: `false`.
+
 ### skip_invalid_rows (bool) (optional)
 
 Treat request as success, even if invalid rows exist

data/fluent-plugin-cassandra-json.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-cassandra-json"
-  spec.version = "0.1.3"
+  spec.version = "0.2.0"
   spec.authors = ["joker1007"]
   spec.email   = ["kakyoin.hierophant@gmail.com"]
 

data/lib/fluent/plugin/out_cassandra_json.rb

@@ -48,8 +48,10 @@ module Fluent
         desc: "Use IF NOT EXIST option on INSERT"
       config_param :ttl, :integer, default: nil,
         desc: "Use TTL option on INSERT"
-      config_param :idempotent, :bool, default: false,
+      config_param :idempotent, :bool, default: true,
         desc: "Specify whether this statement can be retried safely on timeout"
+      config_param :default_unset, :bool, default: false,
+        desc: "Specify whether column not defined in the JSON is set to null or is ignored (If false, column not defined in the JSON is set to null. It is cassandra default)"
 
       config_param :skip_invalid_rows, :bool, default: true,
         desc: "Treat request as success, even if invalid rows exist"
@@ -110,7 +112,7 @@ module Fluent
         futures = chunk.open do |io|
           io.map do |line|
             line.chomp!
-            cql = "INSERT INTO #{keyspace}.#{table} JSON '#{line.gsub("'", "''")}'"
+            cql = "INSERT INTO #{keyspace}.#{table} JSON '#{line.gsub("'", "''")}'#{@default_unset ? " DEFAULT UNSET" : ""}"
             cql << " IF NOT EXISTS" if @if_not_exists
             cql << " USING TTL #{@ttl}" if @ttl && @ttl > 0
             @log.debug(cql)

data/test/plugin/test_out_cassandra_json.rb

@@ -148,6 +148,94 @@ class CassandraJsonOutputTest < Test::Unit::TestCase
     end
   end
 
+  test "write data (default_unset is false)" do
+    conf = %[
+      hosts #{ENV.fetch("CASSANDRA_HOST", "127.0.0.1")}
+      port #{ENV.fetch("CASSANDRA_PORT", "9042")}
+      keyspace test_keyspace
+      table test_table
+      default_unset false
+    ]
+    driver = create_driver(conf)
+    time = Time.local(2018, 5, 8, 14, 10, 5)
+    data = {
+      "id" => 1,
+      "col1" => "textdata",
+      "col2" => time,
+      "col3" => true,
+      "col4" => 1.23,
+      "col5" => [1, 2, 3],
+      "col6" => ["one", "two", "three"],
+      "col7" => {"key1" => "val1", "key2" => "val2"},
+    }
+    data2 = {
+      "id" => 1,
+      "col1" => "textdata2",
+    }
+    driver.run do
+      driver.feed("tag", Time.now.to_i, data)
+      driver.flush
+      result = @session.execute("SELECT * FROM test_keyspace.test_table")
+      assert { result.size == 1 }
+      driver.feed("tag", Time.now.to_i, data2)
+    end
+
+    result = @session.execute("SELECT * FROM test_keyspace.test_table")
+    assert { result.size == 1 }
+
+    first = result.each.to_a[0]
+    assert { first["id"] == 1 }
+    assert { first["col1"] == "textdata2" }
+    (2..7).each do |i|
+      assert { first["col#{i}"].nil? }
+    end
+  end
+
+  test "write data (default_unset is true)" do
+    conf = %[
+      hosts #{ENV.fetch("CASSANDRA_HOST", "127.0.0.1")}
+      port #{ENV.fetch("CASSANDRA_PORT", "9042")}
+      keyspace test_keyspace
+      table test_table
+      default_unset true
+    ]
+    driver = create_driver(conf)
+    time = Time.local(2018, 5, 8, 14, 10, 5)
+    data = {
+      "id" => 1,
+      "col1" => "textdata",
+      "col2" => time,
+      "col3" => true,
+      "col4" => 1.23,
+      "col5" => [1, 2, 3],
+      "col6" => ["one", "two", "three"],
+      "col7" => {"key1" => "val1", "key2" => "val2"},
+    }
+    data2 = {
+      "id" => 1,
+      "col1" => "textdata2",
+    }
+    driver.run do
+      driver.feed("tag", Time.now.to_i, data)
+      driver.flush
+      result = @session.execute("SELECT * FROM test_keyspace.test_table")
+      assert { result.size == 1 }
+      driver.feed("tag", Time.now.to_i, data2)
+    end
+
+    result = @session.execute("SELECT * FROM test_keyspace.test_table")
+    assert { result.size == 1 }
+
+    first = result.each.to_a[0]
+    expected = data.dup
+    expected["col1"] = "textdata2"
+    expected["col6"] = Set.new(data["col6"])
+    assert { first["id"] == expected["id"] }
+    (1..7).each do |i|
+      assert { first["col#{i}"] == expected["col#{i}"] }
+    end
+  end
+
   private
 
   def create_driver(conf = CONFIG)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cassandra-json
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-26 00:00:00.000000000 Z
+date: 2018-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler