fluent-plugin-dynamodb-alt 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 313cdebb4acdd110160053ae3e62fae21df1f82e
-  data.tar.gz: 42001e619859094c6e118e511310c71edb47b8ce
+  metadata.gz: def6475b444e76ca66d619f4fea1f814a4c64431
+  data.tar.gz: 3f83b0bd39fe53f813f2b8273b5e2e39392145da
 SHA512:
-  metadata.gz: 9b5f25c37dd794f4f7500b55d4c6c3ad2b3f7e6712c90002129d84657381694924d7db4e8900aa9607fd0cf7a80ec2a0f21407cf893ed924a712b7702079e080
-  data.tar.gz: 863bb9e03764a4cab2d66629881c701b10be1bd2a07a70e52e8a94af18a1a28fd4db4803a667dfb89856e5435cbb1fee1e48a98a67d36e406400d326caa5e795
+  metadata.gz: 552cd086584a6edcf30139cb6b6ea4b9081c2913592cd1381e3abe360a1f52958b4a0a4e2f62597d16ee4bc2a98c46536047a552403c792fbcf1f55e5fd04b07
+  data.tar.gz: 6eab1ab0f0540f685264fb2ec7190e2c177ebd7a41ea31dbb149b09aa63390fe17e3727fb2279dc00eb7310cebcb0da392b2c3796c3569407524fb47aa795cec

data/README.md

@@ -29,6 +29,7 @@ bundle exec rake install
   region ap-northeast-1
   table_name my_table
   timestamp_key timestamp
+  #binary_keys data1,data2
   #endpoint http:://localhost:4567
   #concurrency 1
 

data/fluent-plugin-dynamodb-alt.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-dynamodb-alt'
-  spec.version       = '0.1.1'
+  spec.version       = '0.1.2'
   spec.authors       = ['Genki Sugawara']
   spec.email         = ['sgwr_dts@yahoo.co.jp']
   spec.summary       = %q{Fluent plugin to output to DynamoDB.}
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '>= 3.0.0'
   spec.add_development_dependency 'hashie'
   spec.add_development_dependency 'ddbcli'
+  spec.add_development_dependency 'msgpack'
 end

data/lib/fluent/plugin/out_dynamodb_alt.rb

@@ -16,6 +16,7 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
   config_param :endpoint,             :string,  :default => nil
   config_param :table_name,           :string
   config_param :timestamp_key,        :string
+  config_param :binary_keys,          :string,  :default => nil
   config_param :concurrency,          :integer, :default => 1
   config_param :expected,             :string,  :default => nil
   config_param :conditional_operator, :string,  :default => 'AND'
@@ -27,6 +28,7 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
     super
     require 'aws-sdk-core'
     require 'parallel'
+    require 'stringio'
   end
 
   def configure(conf)
@@ -66,6 +68,12 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
       @expected = parse_expected(@expected)
       log.info("dynamodb_alt expected: #{@expected.inspect}")
     end
+
+    if @binary_keys
+      @binary_keys = @binary_keys.strip.split(/\s*,\s*/)
+    else
+      @binary_keys = []
+    end
   end
 
   def start
@@ -102,6 +110,8 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
   end
 
   def put_record(record)
+    convert_binary!(record)
+
     item = {
       :table_name => @table_name,
       :item => record
@@ -201,4 +211,13 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
       }.last
     }
   end
+
+  def convert_binary!(record)
+    @binary_keys.each do |key|
+      val = record[key]
+      record[key] = StringIO.new(val) if val
+    end
+
+    return record
+  end
 end

data/spec/out_dynamodb_alt_spec.rb

@@ -32,6 +32,7 @@ describe Fluent::DynamodbAltOutput do
         endpoint http:://localhost:4567
         table_name my_table
         timestamp_key timestamp
+        binary_keys aaa,bbb
         concurrency 2
         conditional_operator OR
       EOS
@@ -42,12 +43,37 @@ describe Fluent::DynamodbAltOutput do
       expect(driver.instance.endpoint            ).to eq 'http:://localhost:4567'
       expect(driver.instance.table_name          ).to eq 'my_table'
       expect(driver.instance.timestamp_key       ).to eq 'timestamp'
+      expect(driver.instance.binary_keys         ).to eq ['aaa', 'bbb']
       expect(driver.instance.concurrency         ).to eq 2
       expect(driver.instance.conditional_operator).to eq 'OR'
       expect(driver.instance.instance_variable_get(:@hash_key) ).to eq 'hash_key'
       expect(driver.instance.instance_variable_get(:@range_key)).to eq 'range_key'
     end
 
+    it do
+      driver = create_driver
+      allow(driver.instance).to receive(:configure_aws)
+
+      allow(driver.instance).to receive(:create_client) {
+        client = double('client')
+        allow(client).to receive(:describe_table) {
+          Hashie::Mash.new(:table => {
+            :key_schema => [
+              {:key_type => 'HASH',  :attribute_name => 'hash_key'},
+            ]})
+        }
+        client
+      }
+
+      driver.configure(<<-EOS)
+        type dynamodb_alt
+        table_name my_table
+        timestamp_key timestamp
+      EOS
+
+      expect(driver.instance.binary_keys).to eq []
+    end
+
     it do
       driver = create_driver
       allow(driver.instance).to receive(:configure_aws)
@@ -70,7 +96,7 @@ describe Fluent::DynamodbAltOutput do
         expected timestamp GE 0,key LT 100
       EOS
 
-      expected = driver.instance.instance_variable_get(:@expected)
+      expected = driver.instance.expected
       expect(expected).to eq [["timestamp", "GE", 0],["key", "LT", 100]]
     end
 
@@ -96,7 +122,7 @@ describe Fluent::DynamodbAltOutput do
         expected id NULL,timestamp LT ${ts},key EQ ${k}
       EOS
 
-      expected = driver.instance.instance_variable_get(:@expected)
+      expected = driver.instance.expected
 
       expect(expected[0]).to eq ["id", "NULL", nil]
 
@@ -158,7 +184,7 @@ describe Fluent::DynamodbAltOutput do
         expected key1 EQ "str",key2 EQ 1
       EOS
 
-      expected = driver.instance.instance_variable_get(:@expected)
+      expected = driver.instance.expected
       expect(expected).to eq [["key1", "EQ", "str"], ["key2", "EQ", 1]]
     end
   }
@@ -430,5 +456,28 @@ describe Fluent::DynamodbAltOutput do
         end
       end
     }
+
+    context('binary') {
+      it do
+        run_driver(:binary_keys => 'data') do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'data' => MessagePack.pack({'foo' => 100, 'bar' => 'Zzz...'})}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002, 'data' => MessagePack.pack({'foo' => 200, 'bar' => 'Zzz..'})}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003, 'data' => MessagePack.pack({'foo' => 300, 'bar' => 'Zzz.'})}, time)
+        end
+
+        rows = select_all.map do |row|
+          data = row['data']
+          data = Base64.strict_decode64(data)
+          row['data'] = MessagePack.unpack(data)
+          row
+        end
+
+        expect(rows).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001, "data" => {'foo' => 100, 'bar' => 'Zzz...'}},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002, "data" => {'foo' => 200, 'bar' => 'Zzz..'}},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003, "data" => {'foo' => 300, 'bar' => 'Zzz.'}},
+        ]
+      end
+    }
   }
 end

data/spec/spec_helper.rb

@@ -1,7 +1,10 @@
 require 'fluent/test'
 require 'fluent/plugin/out_dynamodb_alt'
+require 'base64'
 require 'hashie'
+require 'msgpack'
 require 'securerandom'
+require 'stringio'
 
 DRIVER_DEFAULT_TAG = 'test.default'
 TEST_TABLE_NAME = 'my_table-' + SecureRandom.uuid

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dynamodb-alt
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Genki Sugawara
@@ -122,6 +122,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluent plugin to output to DynamoDB.
 email:
 - sgwr_dts@yahoo.co.jp