fluent-plugin-kinesis-aggregation 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5337708f7ca323b6ed1e9c47809a6af9f489b885
-  data.tar.gz: 0ed78adebd0c6d2136ccafd2cebc287ae88d86ed
+  metadata.gz: ce0b6aadd6a6326dfe20ac3db8c472eee51b3a51
+  data.tar.gz: b0a194b9450d3c16ea4ac8c9d8bae05bb68fdc8f
 SHA512:
-  metadata.gz: 63145534371f0eb39b3230a77218536d65a0fdbc34ca60f50b72f6377a292c5cbe26ece61399e2eb4d3c429e8193d51d5de2ba850845927b92ffcdb5ab084ea4
-  data.tar.gz: 26c464ef240db8b0801345ca572055995d327a5200b799081b9bb8a3eaaa91255a8adcd59d0f62d8252e055328ef37535a6182dc026dbb07042a793b0d829ee8
+  metadata.gz: b3ee690dc4ac1075c3e484d268c83c73f48f61b0d87f93b90c1884a6ae727f37cc2084710f287284bd59c66cd654a756c1a2f13f186c3c96aa9f4c4ad4b117f8
+  data.tar.gz: 242726a535b0387042718b4ae8b73def2e65cd4dc0d90aaa80d6a986e7a26c120362b38dc5f69c831357db92df366d9a9d78610134002dffd7b494f690da8f4f

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 0.2.0
+
+- switch to google protobuf library (ruby native one uses too much memory)
+
 ## 0.1.1
 
 - fix up conflict with fluent-kinesis plugin

data/fluent-plugin-kinesis-aggregation.gemspec

@@ -17,7 +17,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-kinesis-aggregation"
-  spec.version       = '0.1.1'
+  spec.version       = '0.2.0'
   spec.author        = 'Someone'
   spec.summary       = %q{Fluentd output plugin that sends KPL style aggregated events to Amazon Kinesis.}
   spec.homepage      = "https://github.com/atlassian/fluent-plugin-kinesis-aggregation"
@@ -36,5 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "fluentd", ">= 0.10.53", "< 0.13"
   spec.add_dependency "aws-sdk-core", ">= 2.0.12", "< 3.0"
   spec.add_dependency "msgpack", ">= 0.5.8"
-  spec.add_dependency "protobuf", ">= 3.5.5"
+  spec.add_dependency "google-protobuf", ">= 3.0.0.alpha.4.0"
 end

data/kpl_aggregated_records.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+message AggregatedRecord {
+  repeated string partition_key_table     = 1;
+  repeated string explicit_hash_key_table = 2;
+  repeated Record records                 = 3;
+}
+
+message Tag {
+  string key   = 1;
+  string value = 2;
+}
+
+message Record {
+  uint64 partition_key_index     = 1;
+  uint64 explicit_hash_key_index = 2;
+  bytes  data                    = 3;
+  repeated Tag    tags                    = 4;
+}

data/lib/fluent/plugin/out_kinesis-aggregation.rb

@@ -17,32 +17,29 @@ require 'logger'
 require 'securerandom'
 require 'digest'
 
-require 'protobuf'
-require 'protobuf/message'
+require 'google/protobuf'
 
-
-# https://github.com/awslabs/amazon-kinesis-producer/blob/master/aggregation-format.md
-class AggregatedRecord < ::Protobuf::Message; end
-class Tag < ::Protobuf::Message; end
-class Record < ::Protobuf::Message; end
-
-class AggregatedRecord
-  repeated :string, :partition_key_table, 1
-  repeated :string, :explicit_hash_key_table, 2
-  repeated ::Record, :records, 3
-end
-
-class Tag
-  required :string, :key, 1
-  optional :string, :value, 2
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "AggregatedRecord" do
+    repeated :partition_key_table, :string, 1
+    repeated :explicit_hash_key_table, :string, 2
+    repeated :records, :message, 3, "Record"
+  end
+  add_message "Tag" do
+    optional :key, :string, 1
+    optional :value, :string, 2
+  end
+  add_message "Record" do
+    optional :partition_key_index, :uint64, 1
+    optional :explicit_hash_key_index, :uint64, 2
+    optional :data, :bytes, 3
+    repeated :tags, :message, 4, "Tag"
+  end
 end
 
-class Record
-  required :uint64, :partition_key_index, 1
-  optional :uint64, :explicit_hash_key_index, 2
-  required :bytes, :data, 3
-  repeated ::Tag, :tags, 4
-end
+AggregatedRecord = Google::Protobuf::DescriptorPool.generated_pool.lookup("AggregatedRecord").msgclass
+Tag = Google::Protobuf::DescriptorPool.generated_pool.lookup("Tag").msgclass
+Record = Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgclass
 
 
 module FluentPluginKinesisAggregation
@@ -104,12 +101,12 @@ module FluentPluginKinesisAggregation
     end
 
     def format(tag, time, record)
-      return AggregatedRecord.new(
+      return AggregatedRecord.encode(AggregatedRecord.new(
         records: [Record.new(
-          partition_key_index: 0,
-          data: Yajl.dump(record)
+          partition_key_index: 1,
+          data: Yajl.dump(record).b
         )]
-      ).encode
+      ))
     end
 
     def write(chunk)
@@ -125,9 +122,17 @@ module FluentPluginKinesisAggregation
       # it's valid (in this case) to concatenate the AggregatedRecords
       # to form one AggregatedRecord, since we only have a repeated field
       # in records.
-      message = AggregatedRecord.new(
-        partition_key_table: [partition_key]
-      ).encode + records
+      #
+      # ALSO, since we use google's protobuf stuff (much better
+      # memory usage due to C extension), we're stuck on proto3.
+      # Unfortunately, KPL uses proto2 form, and partition_key_index
+      # is a required field. If we set it to 0 in proto3, though,
+      # it's helpfully ignored in the serialisation (default!).
+      # Therefore we have to pass a partition_key_index of 1,
+      # and put two things in our partition_key_table.
+      message = AggregatedRecord.encode(AggregatedRecord.new(
+        partition_key_table: ['a', partition_key]
+      )) + records
 
       @client.put_record(
         stream_name: @stream_name,

data/test/plugin/test_out_kinesis-aggregation.rb

@@ -163,22 +163,20 @@ class KinesisOutputTest < Test::Unit::TestCase
     d.run
   end
 
-  def test_format
+  def test_emitting
     d = create_driver
 
-    data1 = {"test_partition_key"=>"key1","a"=>1,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
-    data2 = {"test_partition_key"=>"key2","a"=>2,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
+    data1 = {"a"=>1,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
+    data2 = {"a"=>2,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     d.emit(data1, time)
     d.emit(data2, time)
-    d.expect_format("\u001AR\b\u0000\u001AN{\"test_partition_key\":\"key1\",\"a\":1,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}")
-    d.expect_format("\u001AR\b\u0000\u001AN{\"test_partition_key\":\"key2\",\"a\":2,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}")
 
     client = create_mock_client
     client.put_record(
       stream_name: 'test_stream',
-      data: "\xF3\x89\x9A\xC2\n\x12test_partition_key\x1AR\b\x00\x1AN{\"test_partition_key\":\"key1\",\"a\":1,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\x1AR\b\x00\x1AN{\"test_partition_key\":\"key2\",\"a\":2,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\xB6j\x1E\xF7q\xC9}v\vU\xAD\xA3@<\x82\xA9".force_encoding("ASCII-8BIT"),
+      data: "\xF3\x89\x9A\xC2\n\x01a\n\x12test_partition_key\x1A6\b\x01\x1A2{\"a\":1,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\x1A6\b\x01\x1A2{\"a\":2,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\xA2\x0E y\x8B\x02\xDF\xAE\xAB\x93\x1C;\xCB\xAD\x1Fx".b,
       partition_key: 'test_partition_key'
     ) { {} }
 
@@ -188,20 +186,15 @@ class KinesisOutputTest < Test::Unit::TestCase
   def test_multibyte
     d = create_driver
 
-    data1 = {"test_partition_key"=>"key1","a"=>"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB","time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
-    data1["a"].force_encoding("ASCII-8BIT")
+    data1 = {"a"=>"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB","time"=>"2011-01-02T13:14:15Z".b,"tag"=>"test"}
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     d.emit(data1, time)
 
-    d.expect_format(
-        "\x1Ae\b\x00\x1Aa{\"test_partition_key\":\"key1\",\"a\":\"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB\",\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}".force_encoding("ASCII-8BIT")
-    )
-
     client = create_mock_client
     client.put_record(
       stream_name: 'test_stream',
-      data: "\xF3\x89\x9A\xC2\n\x12test_partition_key\x1Ae\b\x00\x1Aa{\"test_partition_key\":\"key1\",\"a\":\"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB\",\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\xC8\x13{\xFBL_\x8FE\x02\xEEC\xC9_~\xEF(".force_encoding("ASCII-8BIT"),
+      data: "\xF3\x89\x9A\xC2\n\x01a\n\x12test_partition_key\x1AI\b\x01\x1AE{\"a\":\"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB\",\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}_$\x9C\xF9v+pV:g7c\xE3\xF2$\xBA".b,
       partition_key: 'test_partition_key'
     ) { {} }
 
@@ -212,7 +205,7 @@ class KinesisOutputTest < Test::Unit::TestCase
     d = create_driver
 
     d.emit(
-      {"msg": "z" * 1024 * 1024},
+      {"msg" => "z" * 1024 * 1024},
       Time.parse("2011-01-02 13:14:15 UTC").to_i)
     client = dont_allow(Object.new)
     client.put_record

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis-aggregation
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Someone
@@ -107,19 +107,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.5.8
 - !ruby/object:Gem::Dependency
-  name: protobuf
+  name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: 3.0.0.alpha.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: 3.0.0.alpha.4.0
 description: 
 email: 
 executables: []
@@ -136,6 +136,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-kinesis-aggregation.gemspec
+- kpl_aggregated_records.proto
 - lib/fluent/plugin/out_kinesis-aggregation.rb
 - test/helper.rb
 - test/plugin/test_out_kinesis-aggregation.rb