fluent-plugin-kinesis-aggregation 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c7aa063d31e8a8d8b373a723b31cb6c284e4bdd
-  data.tar.gz: c4ca64341656418dd2e197c6d76d1ee40508c77f
+  metadata.gz: b75bac816dfcbedd2d634402e10f2e4c00bb9b0c
+  data.tar.gz: d4b59d91022744f5e9a4dcfe7d8f0e4d6ee89878
 SHA512:
-  metadata.gz: 52f71d9072671d888c774163a0216651987f248c67f2a0c11dd7709bd3326619355b6a2dc7807e545616d3aa4512eebb8161bcc0c3019c1add4b39e0e0228962
-  data.tar.gz: ad1ac4922c2f9e0ef39f9f4e0314b034e6a0c7f925511cab377f28574559018e33b7d0dbafe8acd40728ab884879b705eee274c1033b0f07137b757fde5c6182
+  metadata.gz: 0d06cdd97c38fdfa5a6f6ae66547718a4fd7d5d93453a9d2f3c7d749fd0fc93e6ab2d148c791815ab3e289bce4f8f1c00a4f944af938b4149a9fe6653b44685b
+  data.tar.gz: 8bb20bed570042d2619ec72a808b3a046d50b6a14065618b88294d55ed94d951fd5540646f6ac2321c483cd68ab19dd389e9cc701ec8b0732bd37985d7e358d5

data/.travis.yml

@@ -1,6 +1,8 @@
 language: ruby
 
 rvm:
+  - 2.4
+  - 2.3
   - 2.2
   - 2.1
 

data/CHANGELOG.md

@@ -1,9 +1,19 @@
 # CHANGELOG
 
-## Next
+## 0.3.0
+
+- Update to use fluentd 0.14 API (stick to 0.2.3 if you need support for earlier versions of fluentd)
+  Much thanks to cosmo0920 for doing this.
+
+## 0.2.3
+
+- emit stream name in error
+
+## 0.2.1 - 0.2.2
 
 - update documentation to refer to published gem
 - turn on testing for Ruby 2.1
+- allow running on Ruby 2.1
 
 ## 0.2.0
 

data/README.md

@@ -145,24 +145,28 @@ forces all writes to a specific shard, and if you're using
 a single thread/process will probably keep event ordering
 (not recommended - watch out for hot shards!).
 
-### detach_process
-
-Integer. Optional. This defines the number of parallel processes to start.
-This can be used to increase throughput by allowing multiple processes to
-execute the plugin at once. Setting this option to > 0 will cause the plugin
-to run in a separate process. The default is 0.
-
 ### num_threads
 
 Integer. The number of threads to flush the buffer. This plugin is based on
-Fluentd::BufferedOutput, so we buffer incoming records before emitting them to
+Fluentd::Plugin::Output, so we buffer incoming records before emitting them to
 Amazon Kinesis. You can find the detail about buffering mechanism [here](http://docs.fluentd.org/articles/buffer-plugin-overview).
 Emitting records to Amazon Kinesis via network causes I/O Wait, so parallelizing
 emitting with threads will improve throughput.
 
 This option can be used to parallelize writes into the output(s)
 designated by the output plugin. The default is 1.
-Also you can use this option with *detach_process*.
+Also you can use this option with *multi workers*.
+
+### multi workers
+
+This feature is introduced in Fluentd v0.14.
+Instead of using *detach_process*, this feature can use as the following system directive.
+Note that *detach_process* parameter is removed after using v0.14 Output Plugin API.
+The default is 1.
+
+    <system>
+      workers 5
+    </system>
 
 ### debug
 

data/fluent-plugin-kinesis-aggregation.gemspec

@@ -17,11 +17,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-kinesis-aggregation"
-  spec.version       = '0.2.3'
+  spec.version       = '0.3.0'
   spec.author        = 'Atlassian'
+  spec.email         = 'jhaggerty@atlassian.com'
   spec.summary       = %q{Fluentd output plugin that sends KPL style aggregated events to Amazon Kinesis.}
   spec.homepage      = "https://github.com/atlassian/fluent-plugin-kinesis-aggregation"
-  spec.license       = "Apache License, Version 2.0"
+  spec.license       = "Apache-2.0"
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
@@ -33,8 +34,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "test-unit-rr", "~> 1.0"
 
-  spec.add_dependency "fluentd", ">= 0.10.53", "< 0.13"
+  spec.add_dependency "fluentd", [">= 0.14.15", "< 2"]
   spec.add_dependency "aws-sdk-core", ">= 2.0.12", "< 3.0"
-  spec.add_dependency "msgpack", ">= 0.5.8"
-  spec.add_dependency "google-protobuf", ">= 3.0.0.alpha.4.0"
+  spec.add_dependency "msgpack", "~> 1.0"
+  spec.add_dependency "google-protobuf", "~> 3.0"
 end

data/lib/fluent/plugin/out_kinesis-aggregation.rb

@@ -18,6 +18,7 @@ require 'securerandom'
 require 'digest'
 
 require 'google/protobuf'
+require 'fluent/plugin/output'
 
 Google::Protobuf::DescriptorPool.generated_pool.build do
   add_message "AggregatedRecord" do
@@ -43,12 +44,11 @@ Record = Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgcla
 
 
 module FluentPluginKinesisAggregation
-  class OutputFilter < Fluent::BufferedOutput
+  class OutputFilter < Fluent::Plugin::Output
 
-    include Fluent::DetachMultiProcessMixin
-    include Fluent::SetTimeKeyMixin
-    include Fluent::SetTagKeyMixin
+    helpers :compat_parameters, :inject
 
+    DEFAULT_BUFFER_TYPE = "memory"
     NAME = 'kinesis-aggregation'
     PUT_RECORD_MAX_DATA_SIZE = 1024 * 1024
     # 200 is an arbitrary number more than the envelope overhead
@@ -81,26 +81,31 @@ module FluentPluginKinesisAggregation
 
     config_param :http_proxy, :string, default: nil
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject)
       super
 
-      if @buffer.chunk_limit > FLUENTD_MAX_BUFFER_SIZE
+      if @buffer.chunk_limit_size > FLUENTD_MAX_BUFFER_SIZE
         raise Fluent::ConfigError, "Kinesis buffer_chunk_limit is set to more than the 1mb shard limit (i.e. you won't be able to write your chunks!"
       end
 
-      if @buffer.chunk_limit > FLUENTD_MAX_BUFFER_SIZE / 3
+      if @buffer.chunk_limit_size > FLUENTD_MAX_BUFFER_SIZE / 3
         log.warn 'Kinesis buffer_chunk_limit is set at more than 1/3 of the per second shard limit (1mb). This is not good if you have many producers.'
       end
     end
 
     def start
-      detach_multi_process do
-        super
-        load_client
-      end
+      super
+      load_client
     end
 
     def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+
       return AggregatedRecord.encode(AggregatedRecord.new(
         records: [Record.new(
           partition_key_index: 1,

data/test/helper.rb

@@ -28,5 +28,7 @@ require 'test/unit/rr'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
 require 'fluent/process'
 require 'fluent/plugin/out_kinesis-aggregation'

data/test/plugin/test_out_kinesis-aggregation.rb

@@ -14,6 +14,8 @@
 require 'helper'
 
 class KinesisOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
   end
@@ -27,14 +29,14 @@ class KinesisOutputTest < Test::Unit::TestCase
     buffer_chunk_limit 100k
   ]
 
-  def create_driver(conf = CONFIG, tag='test')
-    Fluent::Test::BufferedOutputTestDriver
-      .new(FluentPluginKinesisAggregation::OutputFilter, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output
+      .new(FluentPluginKinesisAggregation::OutputFilter).configure(conf)
   end
 
   def create_mock_client
     client = mock(Object.new)
-    mock(Aws::Kinesis::Client).new(anything) { client }
+    stub(Aws::Kinesis::Client).new(anything) { client }
     return client
   end
 
@@ -104,7 +106,7 @@ class KinesisOutputTest < Test::Unit::TestCase
     end
 
     d = create_driver
-    d.run
+    d.run(default_tag: "test")
   end
 
   def test_load_client_with_credentials
@@ -132,7 +134,7 @@ class KinesisOutputTest < Test::Unit::TestCase
       buffer_chunk_limit 100k
     EOS
 
-    d.run
+    d.run(default_tag: "test")
   end
 
   def test_load_client_with_role_arn
@@ -160,7 +162,7 @@ class KinesisOutputTest < Test::Unit::TestCase
       fixed_partition_key test_partition_key
       buffer_chunk_limit 100k
     EOS
-    d.run
+    d.run(default_tag: "test")
   end
 
   def test_emitting
@@ -169,18 +171,19 @@ class KinesisOutputTest < Test::Unit::TestCase
     data1 = {"a"=>1,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
     data2 = {"a"=>2,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit(data1, time)
-    d.emit(data2, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
 
-    client = create_mock_client
-    client.put_record(
-      stream_name: 'test_stream',
-      data: "\xF3\x89\x9A\xC2\n\x01a\n\x12test_partition_key\x1A6\b\x01\x1A2{\"a\":1,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\x1A6\b\x01\x1A2{\"a\":2,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\xA2\x0E y\x8B\x02\xDF\xAE\xAB\x93\x1C;\xCB\xAD\x1Fx".b,
-      partition_key: 'test_partition_key'
-    ) { {} }
+    d.run(default_tag: "test") do
+      client = create_mock_client
+      stub.instance_of(Aws::Kinesis::Client).put_record(
+        stream_name: 'test_stream',
+        data: "\xF3\x89\x9A\xC2\n\x01a\n\x12test_partition_key\x1A6\b\x01\x1A2{\"a\":1,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\x1A6\b\x01\x1A2{\"a\":2,\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}\xA2\x0E y\x8B\x02\xDF\xAE\xAB\x93\x1C;\xCB\xAD\x1Fx".b,
+        partition_key: 'test_partition_key'
+      ) { {} }
 
-    d.run
+      d.feed(time, data1)
+      d.feed(time, data2)
+    end
   end
 
   def test_multibyte
@@ -188,29 +191,32 @@ class KinesisOutputTest < Test::Unit::TestCase
 
     data1 = {"a"=>"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB","time"=>"2011-01-02T13:14:15Z".b,"tag"=>"test"}
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit(data1, time)
 
-    client = create_mock_client
-    client.put_record(
-      stream_name: 'test_stream',
-      data: "\xF3\x89\x9A\xC2\n\x01a\n\x12test_partition_key\x1AI\b\x01\x1AE{\"a\":\"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB\",\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}_$\x9C\xF9v+pV:g7c\xE3\xF2$\xBA".b,
-      partition_key: 'test_partition_key'
-    ) { {} }
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      client = create_mock_client
+      stub.instance_of(Aws::Kinesis::Client).put_record(
+        stream_name: 'test_stream',
+        data: "\xF3\x89\x9A\xC2\n\x01a\n\x12test_partition_key\x1AI\b\x01\x1AE{\"a\":\"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB\",\"time\":\"2011-01-02T13:14:15Z\",\"tag\":\"test\"}_$\x9C\xF9v+pV:g7c\xE3\xF2$\xBA".b,
+        partition_key: 'test_partition_key'
+      ) { {} }
 
-    d.run
+      d.feed(time, data1)
+    end
   end
 
   def test_fail_on_bigchunk
     d = create_driver
 
-    d.emit(
-      {"msg" => "z" * 1024 * 1024},
-      Time.parse("2011-01-02 13:14:15 UTC").to_i)
-    client = dont_allow(Object.new)
-    client.put_record
-    mock(Aws::Kinesis::Client).new(anything) { client }
-
-    d.run
+    assert_raise(Fluent::Plugin::Buffer::BufferChunkOverflowError) do
+      d.run(default_tag: "test") do
+        d.feed(
+          event_time("2011-01-02 13:14:15 UTC"),
+          {"msg" => "z" * 1024 * 1024})
+        client = dont_allow(Object.new)
+        client.put_record
+        mock(Aws::Kinesis::Client).new(anything) { client }
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis-aggregation
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Atlassian
@@ -58,20 +58,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.53
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.53
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
@@ -96,32 +96,32 @@ dependencies:
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.8
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.8
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0.alpha.4.0
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0.alpha.4.0
+        version: '3.0'
 description: 
-email: 
+email: jhaggerty@atlassian.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -142,7 +142,7 @@ files:
 - test/plugin/test_out_kinesis-aggregation.rb
 homepage: https://github.com/atlassian/fluent-plugin-kinesis-aggregation
 licenses:
-- Apache License, Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []