fluent-plugin-sqs 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7d738c77ff00141662a60dbe4db65109c7c873b
-  data.tar.gz: f399c2e1ee216a6f84cc7c172e22950a4a3e8b0c
+  metadata.gz: ac806e59bc8e71ad1eecb7cc1f3a6fc0c7107d8c
+  data.tar.gz: 01ee1714e4709cadbe7965453abac2d74f664384
 SHA512:
-  metadata.gz: 6d55dc8fe951b822e8b047fcf29f8dbf1ab07775292afa0b1d9dc621e82e2c8b489a8397488a894a22ef90c78210d1de9506483e9a92b2878616ac7da50a2717
-  data.tar.gz: af34dacba968a84f73422c7fd0c94bf1eb924466cfdba99d1f3c06030cae047ce8739424c1470b94045f157d8053facc701ad3f17700163896ea5d4d4d1ac318
+  metadata.gz: 591e14bd3934e022a5eca41cbf9dd588de7d1976625c2d4eb9a8ae8d32da32eddfa52548a2968d63354473169989d7cc3eb9613713cd960db623e743ed7dd2a8
+  data.tar.gz: 82b30c47720e3daabe83efcba3dbb91ac24f53dc87439926b0afb36c93387d7646f94806267a2a84dfd86b57aab38119c77d9ac0adcdd76955388570dfc265df

data/VERSION

@@ -1 +1 @@
-2.0.0
+2.1.0

data/lib/fluent/plugin/in_sqs.rb

@@ -1,11 +1,11 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'aws-sdk'
 
-module Fluent
+module Fluent::Plugin
   class SQSInput < Input
-    Plugin.register_input('sqs', self)
+    Fluent::Plugin.register_input('sqs', self)
 
-    define_method('router') { Fluent::Engine } unless method_defined?(:router)
+    helpers :timer
 
     config_param :aws_key_id, :string, default: nil, secret: true
     config_param :aws_sec_key, :string, default: nil, secret: true
@@ -32,8 +32,7 @@ module Fluent
     def start
       super
 
-      @finished = false
-      @thread = Thread.new(&method(:run_periodic))
+      timer_execute(:in_sqs_run_periodic_timer, @receive_interval, &method(:run))
     end
 
     def client
@@ -46,16 +45,6 @@ module Fluent
 
     def shutdown
       super
-
-      @finished = true
-      @thread.join
-    end
-
-    def run_periodic
-      until @finished
-        sleep @receive_interval
-        run
-      end
     end
 
     def run
@@ -68,11 +57,11 @@ module Fluent
 
         message.delete if @delete_message
 
-        router.emit(@tag, Time.now.to_i, record)
+        router.emit(@tag, Fluent::Engine.now, record)
       end
     rescue
-      $log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
-      $log.warn_backtrace $ERROR_INFO.backtrace
+      log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
+      log.warn_backtrace $ERROR_INFO.backtrace
     end
 
     private

data/lib/fluent/plugin/out_sqs.rb

@@ -1,17 +1,18 @@
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'aws-sdk'
 
-module Fluent
+module Fluent::Plugin
   SQS_BATCH_SEND_MAX_MSGS = 10
   SQS_BATCH_SEND_MAX_SIZE = 262_144
 
-  class SQSOutput < BufferedOutput
+  class SQSOutput < Output
     Fluent::Plugin.register_output('sqs', self)
 
-    include SetTagKeyMixin
-    config_set_default :include_tag_key, false
+    helpers :compat_parameters, :inject
+
+    DEFAULT_BUFFER_TYPE = "memory"
 
-    include SetTimeKeyMixin
+    config_set_default :include_tag_key, false
     config_set_default :include_time_key, true
 
     config_param :aws_key_id, :string, default: nil, secret: true
@@ -24,7 +25,12 @@ module Fluent
     config_param :include_tag, :bool, default: true
     config_param :tag_property_name, :string, default: '__tag'
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject)
       super
 
       Aws.config = {
@@ -58,12 +64,21 @@ module Fluent
       @queue
     end
 
-    def format(tag, _time, record)
+    def format(tag, time, record)
       record[@tag_property_name] = tag if @include_tag
+      record = inject_values_to_record(tag, time, record)
 
       record.to_msgpack
     end
 
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
       batch_records = []
       batch_size = 0
@@ -98,7 +113,8 @@ module Fluent
     end
 
     def generate_id
-      @tag_property_name + Time.now.to_i.to_s
+      unique_val = ((('a'..'z').to_a + (0..9).to_a)*3).shuffle[0,(rand(10).to_i)].join
+      @tag_property_name + Time.now.to_i.to_s + unique_val
     end
   end
 end

data/spec/lib/fluent/plugin/in_sqs_spec.rb

@@ -1,10 +1,10 @@
 require 'spec_helper'
 require 'fluent/plugin/in_sqs'
 
-describe Fluent::SQSInput do
+describe Fluent::Plugin::SQSInput do
   let(:driver) do
     Fluent::Test.setup
-    Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQSInput).configure(config)
   end
   subject { driver.instance }
 
@@ -85,7 +85,7 @@ describe Fluent::SQSInput do
           .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
         expect(subject).to receive(:parse_message).with(message) { message_attributes }
         expect(message).not_to receive(:delete)
-        expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
+        expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
 
         subject.run
       end
@@ -111,7 +111,7 @@ describe Fluent::SQSInput do
           .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
         expect(subject).to receive(:parse_message).with(message) { message_attributes }
         expect(message).to receive(:delete)
-        expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
+        expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
 
         subject.run
       end

data/spec/lib/fluent/plugin/out_sqs_spec.rb

@@ -1,8 +1,8 @@
 require 'spec_helper'
 require 'fluent/plugin/out_sqs'
 
-describe Fluent::SQSOutput do
-  let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQSOutput) }
+describe Fluent::Plugin::SQSOutput do
+  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::SQSOutput) }
   subject { driver.instance }
 
   before do
@@ -121,9 +121,12 @@ describe Fluent::SQSOutput do
     it 'send_messages to queue' do
       allow(Yajl).to receive(:dump).with(record) { body }
 
+      expect(driver.instance).to receive(:queue).twice.and_return("QUEUE_NAME")
       expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
 
-      driver.emit(record).run
+      driver.run(default_tag: "test") do
+        driver.feed(record)
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -16,6 +16,9 @@ RSpec.configure do |config|
 
   require 'fluent/load'
   require 'fluent/test'
+  require 'fluent/test/helpers'
+  require 'fluent/test/driver/output'
+  require 'fluent/test/driver/input'
 
   require 'ostruct'
   require 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sqs
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Yuri Odagiri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-09 00:00:00.000000000 Z
+date: 2017-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -192,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 3
 summary: Amazon SQS input/output plugin for Fluent event collector