fluent-plugin-sqs 1.8.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f51886fc46c9dff3450a0cd2adb6c3315961ff4f
4
- data.tar.gz: a2f22f1ffec81f241b9cfa572401ce7a8e956cd4
3
+ metadata.gz: c7d738c77ff00141662a60dbe4db65109c7c873b
4
+ data.tar.gz: f399c2e1ee216a6f84cc7c172e22950a4a3e8b0c
5
5
  SHA512:
6
- metadata.gz: fddc35298bdcd515cb3161821804f415dc28a6c4f9eb26ca5696649ea92baf338cce5101d2f6a9fc6fcb3cfe1858b5d00fde640792bf339d3f3ed59a0105d9a2
7
- data.tar.gz: e8b6c5413fd3675cc9b7bffc3febb454287c69cdf1e046eb04aa6efdb144d9b41cade14434deafcd7e12e9674c7ee9ee7b59f077f14a62da5df452ee86454a3d
6
+ metadata.gz: 6d55dc8fe951b822e8b047fcf29f8dbf1ab07775292afa0b1d9dc621e82e2c8b489a8397488a894a22ef90c78210d1de9506483e9a92b2878616ac7da50a2717
7
+ data.tar.gz: af34dacba968a84f73422c7fd0c94bf1eb924466cfdba99d1f3c06030cae047ce8739424c1470b94045f157d8053facc701ad3f17700163896ea5d4d4d1ac318
data/README.rdoc CHANGED
@@ -35,16 +35,16 @@ Read events from from amazon SQS.
35
35
  # following attibutes are optional
36
36
   
37
37
  create_queue {boolean}
38
- sqs_endpoint {endpointURL}
38
+ region {your_region}
39
39
 
40
- ### endpoint list ###
41
- # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
42
- # Asia Pacific (Singapore) : sqs.ap-southeast-1.amazonaws.com
43
- # US-East (Virginia) : sqs.us-east-1.amazonaws.com
44
- # US-West (Oregon) : sqs.us-west-2.amazonaws.com
45
- # US-West (N.California) : sqs.us-west-1.amazonaws.com
46
- # EU-West (Ireland) : sqs.eu-west-1.amazonaws.com
47
- # South America (São Paulo) : sns.sa-east-1.amazonaws.com
40
+ ### region list ###
41
+ # Asia Pacific (Tokyo) [Default] : ap-northeast-1
42
+ # Asia Pacific (Singapore) : ap-southeast-1
43
+ # US-East (Virginia) : us-east-1
44
+ # US-West (Oregon) : us-west-2
45
+ # US-West (N.California) : us-west-1
46
+ # EU-West (Ireland) : eu-west-1
47
+ # South America (São Paulo) : sa-east-1
48
48
 
49
49
  delay_seconds {delivery_delay_seconds}
50
50
 
@@ -71,16 +71,16 @@ Read events from from amazon SQS.
71
71
 
72
72
  # following attibutes are optional
73
73
   
74
- sqs_endpoint {endpointURL}
74
+ region {your_region}
75
75
 
76
- ### endpoint list ###
77
- # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
78
- # Asia Pacific (Singapore) : sqs.ap-southeast-1.amazonaws.com
79
- # US-East (Virginia) : sqs.us-east-1.amazonaws.com
80
- # US-West (Oregon) : sqs.us-west-2.amazonaws.com
81
- # US-West (N.California) : sqs.us-west-1.amazonaws.com
82
- # EU-West (Ireland) : sqs.eu-west-1.amazonaws.com
83
- # South America (São Paulo) : sns.sa-east-1.amazonaws.com
76
+ ### region list ###
77
+ # Asia Pacific (Tokyo) [Default] : ap-northeast-1
78
+ # Asia Pacific (Singapore) : ap-southeast-1
79
+ # US-East (Virginia) : us-east-1
80
+ # US-West (Oregon) : us-west-2
81
+ # US-West (N.California) : us-west-1
82
+ # EU-West (Ireland) : eu-west-1
83
+ # South America (São Paulo) : sa-east-1
84
84
 
85
85
  receive_interval {receive_interval_seconds}
86
86
 
@@ -88,6 +88,8 @@ Read events from from amazon SQS.
88
88
 
89
89
  wait_time_seconds {receive_message_wait_time_seconds}
90
90
 
91
+ delete_message {delete_message_after_receiving}
92
+
91
93
  </source>
92
94
 
93
95
  == Tool
data/Rakefile CHANGED
@@ -1,9 +1,9 @@
1
1
  require 'rake'
2
2
  require 'rake/testtask'
3
3
  require 'rake/clean'
4
- require "bundler/gem_tasks"
5
- require "rspec/core/rake_task"
4
+ require 'bundler/gem_tasks'
5
+ require 'rspec/core/rake_task'
6
6
 
7
- RSpec::Core::RakeTask.new("spec")
7
+ RSpec::Core::RakeTask.new('spec')
8
8
 
9
- task :default => [:build]
9
+ task default: [:build]
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.8.0
1
+ 2.0.0
@@ -1,46 +1,49 @@
1
- module Fluent
2
-
3
- require 'aws-sdk-v1'
1
+ require 'fluent/input'
2
+ require 'aws-sdk'
4
3
 
4
+ module Fluent
5
5
  class SQSInput < Input
6
6
  Plugin.register_input('sqs', self)
7
7
 
8
- unless method_defined?(:router)
9
- define_method("router") { Fluent::Engine }
10
- end
8
+ define_method('router') { Fluent::Engine } unless method_defined?(:router)
11
9
 
12
- def initialize
13
- super
14
- end
15
-
16
- config_param :aws_key_id, :string, :default => nil, :secret => true
17
- config_param :aws_sec_key, :string, :default => nil, :secret => true
10
+ config_param :aws_key_id, :string, default: nil, secret: true
11
+ config_param :aws_sec_key, :string, default: nil, secret: true
18
12
  config_param :tag, :string
19
- config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
20
- config_param :sqs_url, :string
21
- config_param :receive_interval, :time, :default => 0.1
22
- config_param :max_number_of_messages, :integer, :default => 10
23
- config_param :wait_time_seconds, :integer, :default => 10
13
+ config_param :region, :string, default: 'ap-northeast-1'
14
+ config_param :sqs_url, :string, default: nil
15
+ config_param :receive_interval, :time, default: 0.1
16
+ config_param :max_number_of_messages, :integer, default: 10
17
+ config_param :wait_time_seconds, :integer, default: 10
18
+ config_param :visibility_timeout, :integer, default: nil
19
+ config_param :delete_message, :bool, default: false
20
+ config_param :stub_responses, :bool, default: false
24
21
 
25
22
  def configure(conf)
26
23
  super
27
24
 
25
+ Aws.config = {
26
+ access_key_id: @aws_key_id,
27
+ secret_access_key: @aws_sec_key,
28
+ region: @region
29
+ }
28
30
  end
29
31
 
30
32
  def start
31
33
  super
32
34
 
33
- AWS.config(
34
- :access_key_id => @aws_key_id,
35
- :secret_access_key => @aws_sec_key
36
- )
37
-
38
- @queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
39
-
40
35
  @finished = false
41
36
  @thread = Thread.new(&method(:run_periodic))
42
37
  end
43
38
 
39
+ def client
40
+ @client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
41
+ end
42
+
43
+ def queue
44
+ @queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
45
+ end
46
+
44
47
  def shutdown
45
48
  super
46
49
 
@@ -50,27 +53,39 @@ module Fluent
50
53
 
51
54
  def run_periodic
52
55
  until @finished
53
- begin
54
- sleep @receive_interval
55
- @queue.receive_message(
56
- :limit => @max_number_of_messages,
57
- :wait_time_seconds => @wait_time_seconds
58
- ) do |message|
59
- record = {}
60
- record['body'] = message.body.to_s
61
- record['handle'] = message.handle.to_s
62
- record['id'] = message.id.to_s
63
- record['md5'] = message.md5.to_s
64
- record['url'] = message.queue.url.to_s
65
- record['sender_id'] = message.sender_id.to_s
66
-
67
- router.emit(@tag, Time.now.to_i, record)
68
- end
69
- rescue
70
- $log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
71
- $log.warn_backtrace $!.backtrace
72
- end
56
+ sleep @receive_interval
57
+ run
58
+ end
59
+ end
60
+
61
+ def run
62
+ queue.receive_messages(
63
+ max_number_of_messages: @max_number_of_messages,
64
+ wait_time_seconds: @wait_time_seconds,
65
+ visibility_timeout: @visibility_timeout
66
+ ).each do |message|
67
+ record = parse_message(message)
68
+
69
+ message.delete if @delete_message
70
+
71
+ router.emit(@tag, Time.now.to_i, record)
73
72
  end
73
+ rescue
74
+ $log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
75
+ $log.warn_backtrace $ERROR_INFO.backtrace
76
+ end
77
+
78
+ private
79
+
80
+ def parse_message(message)
81
+ {
82
+ 'body' => message.body.to_s,
83
+ 'receipt_handle' => message.receipt_handle.to_s,
84
+ 'message_id' => message.message_id.to_s,
85
+ 'md5_of_body' => message.md5_of_body.to_s,
86
+ 'queue_url' => message.queue_url.to_s,
87
+ 'sender_id' => message.attributes['SenderId'].to_s
88
+ }
74
89
  end
75
90
  end
76
91
  end
@@ -1,98 +1,104 @@
1
- module Fluent
2
-
3
- require 'aws-sdk-v1'
1
+ require 'fluent/output'
2
+ require 'aws-sdk'
4
3
 
5
- SQS_BATCH_SEND_MAX_MSGS = 10
6
- SQS_BATCH_SEND_MAX_SIZE = 262144
4
+ module Fluent
5
+ SQS_BATCH_SEND_MAX_MSGS = 10
6
+ SQS_BATCH_SEND_MAX_SIZE = 262_144
7
+
8
+ class SQSOutput < BufferedOutput
9
+ Fluent::Plugin.register_output('sqs', self)
10
+
11
+ include SetTagKeyMixin
12
+ config_set_default :include_tag_key, false
13
+
14
+ include SetTimeKeyMixin
15
+ config_set_default :include_time_key, true
16
+
17
+ config_param :aws_key_id, :string, default: nil, secret: true
18
+ config_param :aws_sec_key, :string, default: nil, secret: true
19
+ config_param :queue_name, :string, default: nil
20
+ config_param :sqs_url, :string, default: nil
21
+ config_param :create_queue, :bool, default: true
22
+ config_param :region, :string, default: 'ap-northeast-1'
23
+ config_param :delay_seconds, :integer, default: 0
24
+ config_param :include_tag, :bool, default: true
25
+ config_param :tag_property_name, :string, default: '__tag'
26
+
27
+ def configure(conf)
28
+ super
29
+
30
+ Aws.config = {
31
+ access_key_id: @aws_key_id,
32
+ secret_access_key: @aws_sec_key,
33
+ region: @region
34
+ }
35
+ end
7
36
 
8
- class SQSOutput < BufferedOutput
9
- Fluent::Plugin.register_output('sqs', self)
37
+ def client
38
+ @client ||= Aws::SQS::Client.new
39
+ end
10
40
 
11
- include SetTagKeyMixin
12
- config_set_default :include_tag_key, false
41
+ def resource
42
+ @resource ||= Aws::SQS::Resource.new(client: client)
43
+ end
13
44
 
14
- include SetTimeKeyMixin
15
- config_set_default :include_time_key, true
45
+ def queue
46
+ return @queue if @queue
16
47
 
17
- config_param :aws_key_id, :string, :default => nil, :secret => true
18
- config_param :aws_sec_key, :string, :default => nil, :secret => true
19
- config_param :queue_name, :string, :default => nil
20
- config_param :sqs_url, :string, :default => nil
21
- config_param :create_queue, :bool, :default => true
22
- config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
23
- config_param :delay_seconds, :integer, :default => 0
24
- config_param :include_tag, :bool, :default => true
25
- config_param :tag_property_name, :string, :default => '__tag'
48
+ @queue = if @create_queue && @queue_name
49
+ resource.create_queue(queue_name: @queue_name)
50
+ else
51
+ @queue = if @sqs_url
52
+ resource.queue(@sqs_url)
53
+ else
54
+ resource.get_queue_by_name(queue_name: @queue_name)
55
+ end
56
+ end
26
57
 
27
- def configure(conf)
28
- super
29
- end
58
+ @queue
59
+ end
30
60
 
31
- def start
32
- super
61
+ def format(tag, _time, record)
62
+ record[@tag_property_name] = tag if @include_tag
33
63
 
34
- AWS.config(
35
- :access_key_id => @aws_key_id,
36
- :secret_access_key => @aws_sec_key)
64
+ record.to_msgpack
65
+ end
37
66
 
38
- @sqs_endpoint = @sqs_url.gsub(/https:\/\/(.*?)\/.*/, '\1') if @sqs_url and not @sqs_endpoint
67
+ def write(chunk)
68
+ batch_records = []
69
+ batch_size = 0
70
+ send_batches = [batch_records]
39
71
 
40
- @sqs = AWS::SQS.new(
41
- :sqs_endpoint => @sqs_endpoint)
72
+ chunk.msgpack_each do |record|
73
+ body = Yajl.dump(record)
74
+ batch_size += body.bytesize
42
75
 
43
- if @create_queue and @queue_name then
44
- @queue = @sqs.queues.create(@queue_name)
45
- else
46
- if @sqs_url # retrive queue by url
47
- @queue = @sqs.queues[@sqs_url]
48
- else
49
- @queue = @sqs.queues.named(@queue_name)
50
- end
51
- end
76
+ if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
77
+ batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
78
+ batch_records = []
79
+ batch_size = body.bytesize
80
+ send_batches << batch_records
52
81
  end
53
82
 
54
- def shutdown
55
- super
83
+ if batch_size > SQS_BATCH_SEND_MAX_SIZE
84
+ log.warn 'Could not push message to SQS, payload exceeds ' \
85
+ "#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
86
+ "(Truncated message: #{body[0..200]})"
87
+ else
88
+ batch_records << { id: generate_id, message_body: body, delay_seconds: @delay_seconds }
56
89
  end
90
+ end
57
91
 
58
- def format(tag, time, record)
59
- if @include_tag then
60
- record[@tag_property_name] = tag
61
- end
62
-
63
- record.to_msgpack
92
+ until send_batches.length <= 0
93
+ records = send_batches.shift
94
+ until records.length <= 0
95
+ queue.send_messages(entries: records.slice!(0..9))
64
96
  end
97
+ end
98
+ end
65
99
 
66
- def write(chunk)
67
- batch_records = []
68
- batch_size = 0
69
- send_batches = [batch_records]
70
-
71
- chunk.msgpack_each do |record|
72
- body = Yajl.dump(record)
73
- batch_size += body.bytesize
74
- if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
75
- batch_records.length >= SQS_BATCH_SEND_MAX_MSGS then
76
- batch_records = []
77
- batch_size = body.bytesize
78
- send_batches << batch_records
79
- end
80
-
81
- if batch_size > SQS_BATCH_SEND_MAX_SIZE then
82
- log.warn "Could not push message to SQS, payload exceeds "\
83
- "#{SQS_BATCH_SEND_MAX_SIZE} bytes. "\
84
- "(Truncated message: #{body[0..200]})"
85
- else
86
- batch_records << { :message_body => body, :delay_seconds => @delay_seconds }
87
- end
88
- end
89
-
90
- until send_batches.length <= 0 do
91
- records = send_batches.shift
92
- until records.length <= 0 do
93
- @queue.batch_send(records.slice!(0..9))
94
- end
95
- end
96
- end
100
+ def generate_id
101
+ @tag_property_name + Time.now.to_i.to_s
97
102
  end
103
+ end
98
104
  end
@@ -1,113 +1,120 @@
1
1
  require 'spec_helper'
2
+ require 'fluent/plugin/in_sqs'
2
3
 
3
- describe do
4
- let(:driver) {
5
- AWS.stub!
4
+ describe Fluent::SQSInput do
5
+ let(:driver) do
6
6
  Fluent::Test.setup
7
7
  Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
8
- }
9
- let(:instance) {driver.instance}
8
+ end
9
+ subject { driver.instance }
10
10
 
11
- describe 'config' do
12
- let(:config) {
13
- %[
11
+ describe '#configure' do
12
+ let(:config) do
13
+ %(
14
14
  aws_key_id AWS_KEY_ID
15
15
  aws_sec_key AWS_SEC_KEY
16
- tag TAG
17
- sqs_url SQS_URL
16
+ tag TAG
17
+ region REGION
18
+ sqs_url http://SQS_URL
19
+ receive_interval 1
18
20
  max_number_of_messages 10
19
21
  wait_time_seconds 10
20
- ]
21
- }
22
-
23
- context do
24
- subject {instance.aws_key_id}
25
- it{should == 'AWS_KEY_ID'}
26
- end
27
-
28
- context do
29
- subject {instance.aws_sec_key}
30
- it{should == 'AWS_SEC_KEY'}
22
+ visibility_timeout 1
23
+ delete_message true
24
+ stub_responses true
25
+ )
31
26
  end
32
27
 
33
- context do
34
- subject {instance.tag}
35
- it{should == 'TAG'}
28
+ context 'fluentd input configuration settings' do
29
+ it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
30
+ it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
31
+ it { expect(subject.tag).to eq('TAG') }
32
+ it { expect(subject.region).to eq('REGION') }
33
+ it { expect(subject.sqs_url).to eq('http://SQS_URL') }
34
+ it { expect(subject.receive_interval).to eq(1) }
35
+ it { expect(subject.max_number_of_messages).to eq(10) }
36
+ it { expect(subject.wait_time_seconds).to eq(10) }
37
+ it { expect(subject.visibility_timeout).to eq(1) }
38
+ it { expect(subject.delete_message).to eq(true) }
39
+ it { expect(subject.stub_responses).to eq(true) }
36
40
  end
37
41
 
38
- context do
39
- subject {instance.sqs_url}
40
- it{should == 'SQS_URL'}
41
- end
42
-
43
- context do
44
- subject {instance.receive_interval}
45
- it{should == 0.1}
46
- end
42
+ context 'AWS configuration settings' do
43
+ subject { Aws.config }
47
44
 
48
- context do
49
- subject {instance.max_number_of_messages}
50
- it{should == 10}
51
- end
45
+ before { driver.instance }
52
46
 
53
- context do
54
- subject {instance.wait_time_seconds}
55
- it{should == 10}
47
+ it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
48
+ it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
49
+ it { expect(subject[:region]).to eq('REGION') }
56
50
  end
57
51
  end
58
-
59
- describe 'emit' do
60
- let(:message) do
61
- { 'body' => 'body',
62
- 'handle' => 'handle',
63
- 'id' => 'id',
64
- 'md5' => 'md5',
65
- 'url' => 'url',
66
- 'sender_id' => 'sender_id'
52
+
53
+ describe '#run' do
54
+ let(:message_attributes) do
55
+ {
56
+ body: 'body',
57
+ receipt_handle: 'receipt_handle',
58
+ message_id: 'message_id',
59
+ md5_of_body: 'md5_of_body',
60
+ queue_url: 'queue_url',
61
+ attributes: { 'SenderId' => 'sender_id' }
67
62
  }
68
63
  end
69
- let(:emmits) {
70
- allow(Time).to receive(:now).and_return(0)
64
+ let(:queue) { double(:queue, receive_messages: true) }
65
+ let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
66
+ let(:messages) { [message] }
71
67
 
72
- class AWS::SQS::Queue
73
- def receive_message(opts)
74
- yield OpenStruct.new(
75
- { 'body' => 'body',
76
- 'handle' => 'handle',
77
- 'id' => 'id',
78
- 'md5' => 'md5',
79
- 'queue' => OpenStruct.new(:url => 'url'),
80
- 'sender_id' => 'sender_id',
81
- 'sent_at' => 0
82
- })
83
- end
68
+ context 'with no delete messages param' do
69
+ let(:config) do
70
+ %(
71
+ tag TAG
72
+ max_number_of_messages 10
73
+ wait_time_seconds 10
74
+ visibility_timeout 1
75
+ delete_message false
76
+ )
84
77
  end
85
- expect_any_instance_of(AWS::SQS::Queue).to receive(:receive_message).with({:limit => 10, :wait_time_seconds=>10}).at_least(:once).and_call_original
86
78
 
87
- d = driver
88
- d.run do
89
- sleep 2
79
+ before do
80
+ allow(subject).to receive(:queue) { queue }
90
81
  end
91
82
 
92
- d.emits
93
- }
94
-
95
- context do
96
- let(:config) {
97
- %[
98
- aws_key_id AWS_KEY_ID
99
- aws_sec_key AWS_SEC_KEY
100
- tag TAG
101
- sqs_url SQS_URL
102
- max_number_of_messages 10
103
- wait_time_seconds 10
104
- ]
105
- }
83
+ it 'parse through messages and emit it' do
84
+ expect(queue).to receive(:receive_messages)
85
+ .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
86
+ expect(subject).to receive(:parse_message).with(message) { message_attributes }
87
+ expect(message).not_to receive(:delete)
88
+ expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
106
89
 
107
- subject {emmits.first}
108
- it{should == ['TAG', 0, message]}
90
+ subject.run
91
+ end
109
92
  end
110
93
 
111
- end
94
+ context 'with no delete messages param' do
95
+ let(:config) do
96
+ %(
97
+ tag TAG
98
+ max_number_of_messages 10
99
+ wait_time_seconds 10
100
+ visibility_timeout 1
101
+ delete_message true
102
+ )
103
+ end
112
104
 
105
+ before do
106
+ allow(subject).to receive(:queue) { queue }
107
+ end
108
+
109
+ it 'parse through messages and emit it' do
110
+ expect(queue).to receive(:receive_messages)
111
+ .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
112
+ expect(subject).to receive(:parse_message).with(message) { message_attributes }
113
+ expect(message).to receive(:delete)
114
+ expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
115
+
116
+ subject.run
117
+ end
118
+ end
119
+ end
113
120
  end
@@ -0,0 +1,129 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/out_sqs'
3
+
4
+ describe Fluent::SQSOutput do
5
+ let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQSOutput) }
6
+ subject { driver.instance }
7
+
8
+ before do
9
+ Fluent::Test.setup
10
+ driver.configure(config)
11
+ end
12
+
13
+ describe '#configure' do
14
+ let(:config) do
15
+ %(
16
+ aws_key_id AWS_KEY_ID
17
+ aws_sec_key AWS_SEC_KEY
18
+ queue_name QUEUE_NAME
19
+ sqs_url http://SQS_URL
20
+ create_queue false
21
+ region REGION
22
+ delay_seconds 1
23
+ include_tag false
24
+ tag_property_name TAG_PROPERTY_NAME
25
+ )
26
+ end
27
+
28
+ context 'fluentd output configuration settings' do
29
+ it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
30
+ it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
31
+ it { expect(subject.queue_name).to eq('QUEUE_NAME') }
32
+ it { expect(subject.sqs_url).to eq('http://SQS_URL') }
33
+ it { expect(subject.create_queue).to eq(false) }
34
+ it { expect(subject.region).to eq('REGION') }
35
+ it { expect(subject.delay_seconds).to eq(1) }
36
+ it { expect(subject.include_tag).to eq(false) }
37
+ it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
38
+ end
39
+
40
+ context 'AWS configuration settings' do
41
+ subject { Aws.config }
42
+
43
+ before { driver.instance }
44
+
45
+ it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
46
+ it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
47
+ it { expect(subject[:region]).to eq('REGION') }
48
+ end
49
+ end
50
+
51
+ describe '#queue' do
52
+ context 'when create_queue and queue_name are set' do
53
+ let(:config) do
54
+ %(
55
+ queue_name QUEUE_NAME
56
+ create_queue true
57
+ )
58
+ end
59
+ let(:resource_instance) { double(:resource_instance) }
60
+ let(:queue) { double(:queue) }
61
+
62
+ before { allow(subject).to receive(:resource) { resource_instance } }
63
+
64
+ it 'calls on create_queue with queue_name' do
65
+ expect(resource_instance).to receive(:create_queue).with(queue_name: 'QUEUE_NAME') { queue }
66
+
67
+ expect(subject.queue).to eq(queue)
68
+ end
69
+ end
70
+
71
+ context 'when create_queue is not set but a sqs_url is' do
72
+ let(:config) do
73
+ %(
74
+ queue_name QUEUE_NAME
75
+ create_queue false
76
+ sqs_url SQS_URL
77
+ )
78
+ end
79
+ let(:resource_instance) { double(:resource_instance) }
80
+ let(:queue) { double(:queue) }
81
+
82
+ before { allow(subject).to receive(:resource) { resource_instance } }
83
+
84
+ it 'gets queue from sqs_url' do
85
+ expect(resource_instance).to receive(:queue).with('SQS_URL') { queue }
86
+
87
+ expect(subject.queue).to eq(queue)
88
+ end
89
+ end
90
+
91
+ context 'when create_queue is not set nor sqs_url' do
92
+ let(:config) do
93
+ %(
94
+ queue_name QUEUE_NAME
95
+ create_queue false
96
+ )
97
+ end
98
+ let(:resource_instance) { double(:resource_instance) }
99
+ let(:queue) { double(:queue) }
100
+
101
+ before { allow(subject).to receive(:resource) { resource_instance } }
102
+
103
+ it 'gets queue from queue_name' do
104
+ expect(resource_instance).to receive(:get_queue_by_name).with(queue_name: 'QUEUE_NAME') { queue }
105
+
106
+ expect(subject.queue).to eq(queue)
107
+ end
108
+ end
109
+ end
110
+
111
+ describe '#write' do
112
+ let(:config) do
113
+ %(
114
+ queue_name QUEUE_NAME
115
+ )
116
+ end
117
+
118
+ let(:record) { {} }
119
+ let(:body) { double(:body, bytesize: 1) }
120
+
121
+ it 'send_messages to queue' do
122
+ allow(Yajl).to receive(:dump).with(record) { body }
123
+
124
+ expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
125
+
126
+ driver.emit(record).run
127
+ end
128
+ end
129
+ end
data/spec/spec_helper.rb CHANGED
@@ -5,7 +5,6 @@
5
5
  #
6
6
  # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
7
7
  RSpec.configure do |config|
8
- config.treat_symbols_as_metadata_keys_with_true_values = true
9
8
  config.run_all_when_everything_filtered = true
10
9
  config.filter_run :focus
11
10
 
@@ -18,7 +17,6 @@ RSpec.configure do |config|
18
17
  require 'fluent/load'
19
18
  require 'fluent/test'
20
19
 
21
- require 'fluent/plugin/in_sqs'
22
20
  require 'ostruct'
23
21
  require 'pry'
24
22
  require 'rr'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.8.0
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuri Odagiri
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-09 00:00:00.000000000 Z
11
+ date: 2016-06-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -31,19 +31,19 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: aws-sdk-v1
34
+ name: aws-sdk
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ">="
37
+ - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: '0'
39
+ version: '2'
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ">="
44
+ - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: '0'
46
+ version: '2'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: yajl-ruby
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -170,6 +170,7 @@ files:
170
170
  - lib/fluent/plugin/in_sqs.rb
171
171
  - lib/fluent/plugin/out_sqs.rb
172
172
  - spec/lib/fluent/plugin/in_sqs_spec.rb
173
+ - spec/lib/fluent/plugin/out_sqs_spec.rb
173
174
  - spec/spec_helper.rb
174
175
  homepage: https://github.com/ixixi/fluent-plugin-sqs
175
176
  licenses:
@@ -191,10 +192,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
191
192
  version: '0'
192
193
  requirements: []
193
194
  rubyforge_project:
194
- rubygems_version: 2.5.2
195
+ rubygems_version: 2.4.1
195
196
  signing_key:
196
197
  specification_version: 3
197
198
  summary: Amazon SQS input/output plugin for Fluent event collector
198
199
  test_files:
199
200
  - spec/lib/fluent/plugin/in_sqs_spec.rb
201
+ - spec/lib/fluent/plugin/out_sqs_spec.rb
200
202
  - spec/spec_helper.rb