fluent-plugin-sqs 1.8.0 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f51886fc46c9dff3450a0cd2adb6c3315961ff4f
4
- data.tar.gz: a2f22f1ffec81f241b9cfa572401ce7a8e956cd4
3
+ metadata.gz: c7d738c77ff00141662a60dbe4db65109c7c873b
4
+ data.tar.gz: f399c2e1ee216a6f84cc7c172e22950a4a3e8b0c
5
5
  SHA512:
6
- metadata.gz: fddc35298bdcd515cb3161821804f415dc28a6c4f9eb26ca5696649ea92baf338cce5101d2f6a9fc6fcb3cfe1858b5d00fde640792bf339d3f3ed59a0105d9a2
7
- data.tar.gz: e8b6c5413fd3675cc9b7bffc3febb454287c69cdf1e046eb04aa6efdb144d9b41cade14434deafcd7e12e9674c7ee9ee7b59f077f14a62da5df452ee86454a3d
6
+ metadata.gz: 6d55dc8fe951b822e8b047fcf29f8dbf1ab07775292afa0b1d9dc621e82e2c8b489a8397488a894a22ef90c78210d1de9506483e9a92b2878616ac7da50a2717
7
+ data.tar.gz: af34dacba968a84f73422c7fd0c94bf1eb924466cfdba99d1f3c06030cae047ce8739424c1470b94045f157d8053facc701ad3f17700163896ea5d4d4d1ac318
data/README.rdoc CHANGED
@@ -35,16 +35,16 @@ Read events from from amazon SQS.
35
35
  # following attibutes are optional
36
36
   
37
37
  create_queue {boolean}
38
- sqs_endpoint {endpointURL}
38
+ region {your_region}
39
39
 
40
- ### endpoint list ###
41
- # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
42
- # Asia Pacific (Singapore) : sqs.ap-southeast-1.amazonaws.com
43
- # US-East (Virginia) : sqs.us-east-1.amazonaws.com
44
- # US-West (Oregon) : sqs.us-west-2.amazonaws.com
45
- # US-West (N.California) : sqs.us-west-1.amazonaws.com
46
- # EU-West (Ireland) : sqs.eu-west-1.amazonaws.com
47
- # South America (São Paulo) : sns.sa-east-1.amazonaws.com
40
+ ### region list ###
41
+ # Asia Pacific (Tokyo) [Default] : ap-northeast-1
42
+ # Asia Pacific (Singapore) : ap-southeast-1
43
+ # US-East (Virginia) : us-east-1
44
+ # US-West (Oregon) : us-west-2
45
+ # US-West (N.California) : us-west-1
46
+ # EU-West (Ireland) : eu-west-1
47
+ # South America (São Paulo) : sa-east-1
48
48
 
49
49
  delay_seconds {delivery_delay_seconds}
50
50
 
@@ -71,16 +71,16 @@ Read events from from amazon SQS.
71
71
 
72
72
  # following attibutes are optional
73
73
   
74
- sqs_endpoint {endpointURL}
74
+ region {your_region}
75
75
 
76
- ### endpoint list ###
77
- # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
78
- # Asia Pacific (Singapore) : sqs.ap-southeast-1.amazonaws.com
79
- # US-East (Virginia) : sqs.us-east-1.amazonaws.com
80
- # US-West (Oregon) : sqs.us-west-2.amazonaws.com
81
- # US-West (N.California) : sqs.us-west-1.amazonaws.com
82
- # EU-West (Ireland) : sqs.eu-west-1.amazonaws.com
83
- # South America (São Paulo) : sns.sa-east-1.amazonaws.com
76
+ ### region list ###
77
+ # Asia Pacific (Tokyo) [Default] : ap-northeast-1
78
+ # Asia Pacific (Singapore) : ap-southeast-1
79
+ # US-East (Virginia) : us-east-1
80
+ # US-West (Oregon) : us-west-2
81
+ # US-West (N.California) : us-west-1
82
+ # EU-West (Ireland) : eu-west-1
83
+ # South America (São Paulo) : sa-east-1
84
84
 
85
85
  receive_interval {receive_interval_seconds}
86
86
 
@@ -88,6 +88,8 @@ Read events from from amazon SQS.
88
88
 
89
89
  wait_time_seconds {receive_message_wait_time_seconds}
90
90
 
91
+ delete_message {delete_message_after_receiving}
92
+
91
93
  </source>
92
94
 
93
95
  == Tool
data/Rakefile CHANGED
@@ -1,9 +1,9 @@
1
1
  require 'rake'
2
2
  require 'rake/testtask'
3
3
  require 'rake/clean'
4
- require "bundler/gem_tasks"
5
- require "rspec/core/rake_task"
4
+ require 'bundler/gem_tasks'
5
+ require 'rspec/core/rake_task'
6
6
 
7
- RSpec::Core::RakeTask.new("spec")
7
+ RSpec::Core::RakeTask.new('spec')
8
8
 
9
- task :default => [:build]
9
+ task default: [:build]
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.8.0
1
+ 2.0.0
@@ -1,46 +1,49 @@
1
- module Fluent
2
-
3
- require 'aws-sdk-v1'
1
+ require 'fluent/input'
2
+ require 'aws-sdk'
4
3
 
4
+ module Fluent
5
5
  class SQSInput < Input
6
6
  Plugin.register_input('sqs', self)
7
7
 
8
- unless method_defined?(:router)
9
- define_method("router") { Fluent::Engine }
10
- end
8
+ define_method('router') { Fluent::Engine } unless method_defined?(:router)
11
9
 
12
- def initialize
13
- super
14
- end
15
-
16
- config_param :aws_key_id, :string, :default => nil, :secret => true
17
- config_param :aws_sec_key, :string, :default => nil, :secret => true
10
+ config_param :aws_key_id, :string, default: nil, secret: true
11
+ config_param :aws_sec_key, :string, default: nil, secret: true
18
12
  config_param :tag, :string
19
- config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
20
- config_param :sqs_url, :string
21
- config_param :receive_interval, :time, :default => 0.1
22
- config_param :max_number_of_messages, :integer, :default => 10
23
- config_param :wait_time_seconds, :integer, :default => 10
13
+ config_param :region, :string, default: 'ap-northeast-1'
14
+ config_param :sqs_url, :string, default: nil
15
+ config_param :receive_interval, :time, default: 0.1
16
+ config_param :max_number_of_messages, :integer, default: 10
17
+ config_param :wait_time_seconds, :integer, default: 10
18
+ config_param :visibility_timeout, :integer, default: nil
19
+ config_param :delete_message, :bool, default: false
20
+ config_param :stub_responses, :bool, default: false
24
21
 
25
22
  def configure(conf)
26
23
  super
27
24
 
25
+ Aws.config = {
26
+ access_key_id: @aws_key_id,
27
+ secret_access_key: @aws_sec_key,
28
+ region: @region
29
+ }
28
30
  end
29
31
 
30
32
  def start
31
33
  super
32
34
 
33
- AWS.config(
34
- :access_key_id => @aws_key_id,
35
- :secret_access_key => @aws_sec_key
36
- )
37
-
38
- @queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
39
-
40
35
  @finished = false
41
36
  @thread = Thread.new(&method(:run_periodic))
42
37
  end
43
38
 
39
+ def client
40
+ @client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
41
+ end
42
+
43
+ def queue
44
+ @queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
45
+ end
46
+
44
47
  def shutdown
45
48
  super
46
49
 
@@ -50,27 +53,39 @@ module Fluent
50
53
 
51
54
  def run_periodic
52
55
  until @finished
53
- begin
54
- sleep @receive_interval
55
- @queue.receive_message(
56
- :limit => @max_number_of_messages,
57
- :wait_time_seconds => @wait_time_seconds
58
- ) do |message|
59
- record = {}
60
- record['body'] = message.body.to_s
61
- record['handle'] = message.handle.to_s
62
- record['id'] = message.id.to_s
63
- record['md5'] = message.md5.to_s
64
- record['url'] = message.queue.url.to_s
65
- record['sender_id'] = message.sender_id.to_s
66
-
67
- router.emit(@tag, Time.now.to_i, record)
68
- end
69
- rescue
70
- $log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
71
- $log.warn_backtrace $!.backtrace
72
- end
56
+ sleep @receive_interval
57
+ run
58
+ end
59
+ end
60
+
61
+ def run
62
+ queue.receive_messages(
63
+ max_number_of_messages: @max_number_of_messages,
64
+ wait_time_seconds: @wait_time_seconds,
65
+ visibility_timeout: @visibility_timeout
66
+ ).each do |message|
67
+ record = parse_message(message)
68
+
69
+ message.delete if @delete_message
70
+
71
+ router.emit(@tag, Time.now.to_i, record)
73
72
  end
73
+ rescue
74
+ $log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
75
+ $log.warn_backtrace $ERROR_INFO.backtrace
76
+ end
77
+
78
+ private
79
+
80
+ def parse_message(message)
81
+ {
82
+ 'body' => message.body.to_s,
83
+ 'receipt_handle' => message.receipt_handle.to_s,
84
+ 'message_id' => message.message_id.to_s,
85
+ 'md5_of_body' => message.md5_of_body.to_s,
86
+ 'queue_url' => message.queue_url.to_s,
87
+ 'sender_id' => message.attributes['SenderId'].to_s
88
+ }
74
89
  end
75
90
  end
76
91
  end
@@ -1,98 +1,104 @@
1
- module Fluent
2
-
3
- require 'aws-sdk-v1'
1
+ require 'fluent/output'
2
+ require 'aws-sdk'
4
3
 
5
- SQS_BATCH_SEND_MAX_MSGS = 10
6
- SQS_BATCH_SEND_MAX_SIZE = 262144
4
+ module Fluent
5
+ SQS_BATCH_SEND_MAX_MSGS = 10
6
+ SQS_BATCH_SEND_MAX_SIZE = 262_144
7
+
8
+ class SQSOutput < BufferedOutput
9
+ Fluent::Plugin.register_output('sqs', self)
10
+
11
+ include SetTagKeyMixin
12
+ config_set_default :include_tag_key, false
13
+
14
+ include SetTimeKeyMixin
15
+ config_set_default :include_time_key, true
16
+
17
+ config_param :aws_key_id, :string, default: nil, secret: true
18
+ config_param :aws_sec_key, :string, default: nil, secret: true
19
+ config_param :queue_name, :string, default: nil
20
+ config_param :sqs_url, :string, default: nil
21
+ config_param :create_queue, :bool, default: true
22
+ config_param :region, :string, default: 'ap-northeast-1'
23
+ config_param :delay_seconds, :integer, default: 0
24
+ config_param :include_tag, :bool, default: true
25
+ config_param :tag_property_name, :string, default: '__tag'
26
+
27
+ def configure(conf)
28
+ super
29
+
30
+ Aws.config = {
31
+ access_key_id: @aws_key_id,
32
+ secret_access_key: @aws_sec_key,
33
+ region: @region
34
+ }
35
+ end
7
36
 
8
- class SQSOutput < BufferedOutput
9
- Fluent::Plugin.register_output('sqs', self)
37
+ def client
38
+ @client ||= Aws::SQS::Client.new
39
+ end
10
40
 
11
- include SetTagKeyMixin
12
- config_set_default :include_tag_key, false
41
+ def resource
42
+ @resource ||= Aws::SQS::Resource.new(client: client)
43
+ end
13
44
 
14
- include SetTimeKeyMixin
15
- config_set_default :include_time_key, true
45
+ def queue
46
+ return @queue if @queue
16
47
 
17
- config_param :aws_key_id, :string, :default => nil, :secret => true
18
- config_param :aws_sec_key, :string, :default => nil, :secret => true
19
- config_param :queue_name, :string, :default => nil
20
- config_param :sqs_url, :string, :default => nil
21
- config_param :create_queue, :bool, :default => true
22
- config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
23
- config_param :delay_seconds, :integer, :default => 0
24
- config_param :include_tag, :bool, :default => true
25
- config_param :tag_property_name, :string, :default => '__tag'
48
+ @queue = if @create_queue && @queue_name
49
+ resource.create_queue(queue_name: @queue_name)
50
+ else
51
+ @queue = if @sqs_url
52
+ resource.queue(@sqs_url)
53
+ else
54
+ resource.get_queue_by_name(queue_name: @queue_name)
55
+ end
56
+ end
26
57
 
27
- def configure(conf)
28
- super
29
- end
58
+ @queue
59
+ end
30
60
 
31
- def start
32
- super
61
+ def format(tag, _time, record)
62
+ record[@tag_property_name] = tag if @include_tag
33
63
 
34
- AWS.config(
35
- :access_key_id => @aws_key_id,
36
- :secret_access_key => @aws_sec_key)
64
+ record.to_msgpack
65
+ end
37
66
 
38
- @sqs_endpoint = @sqs_url.gsub(/https:\/\/(.*?)\/.*/, '\1') if @sqs_url and not @sqs_endpoint
67
+ def write(chunk)
68
+ batch_records = []
69
+ batch_size = 0
70
+ send_batches = [batch_records]
39
71
 
40
- @sqs = AWS::SQS.new(
41
- :sqs_endpoint => @sqs_endpoint)
72
+ chunk.msgpack_each do |record|
73
+ body = Yajl.dump(record)
74
+ batch_size += body.bytesize
42
75
 
43
- if @create_queue and @queue_name then
44
- @queue = @sqs.queues.create(@queue_name)
45
- else
46
- if @sqs_url # retrive queue by url
47
- @queue = @sqs.queues[@sqs_url]
48
- else
49
- @queue = @sqs.queues.named(@queue_name)
50
- end
51
- end
76
+ if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
77
+ batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
78
+ batch_records = []
79
+ batch_size = body.bytesize
80
+ send_batches << batch_records
52
81
  end
53
82
 
54
- def shutdown
55
- super
83
+ if batch_size > SQS_BATCH_SEND_MAX_SIZE
84
+ log.warn 'Could not push message to SQS, payload exceeds ' \
85
+ "#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
86
+ "(Truncated message: #{body[0..200]})"
87
+ else
88
+ batch_records << { id: generate_id, message_body: body, delay_seconds: @delay_seconds }
56
89
  end
90
+ end
57
91
 
58
- def format(tag, time, record)
59
- if @include_tag then
60
- record[@tag_property_name] = tag
61
- end
62
-
63
- record.to_msgpack
92
+ until send_batches.length <= 0
93
+ records = send_batches.shift
94
+ until records.length <= 0
95
+ queue.send_messages(entries: records.slice!(0..9))
64
96
  end
97
+ end
98
+ end
65
99
 
66
- def write(chunk)
67
- batch_records = []
68
- batch_size = 0
69
- send_batches = [batch_records]
70
-
71
- chunk.msgpack_each do |record|
72
- body = Yajl.dump(record)
73
- batch_size += body.bytesize
74
- if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
75
- batch_records.length >= SQS_BATCH_SEND_MAX_MSGS then
76
- batch_records = []
77
- batch_size = body.bytesize
78
- send_batches << batch_records
79
- end
80
-
81
- if batch_size > SQS_BATCH_SEND_MAX_SIZE then
82
- log.warn "Could not push message to SQS, payload exceeds "\
83
- "#{SQS_BATCH_SEND_MAX_SIZE} bytes. "\
84
- "(Truncated message: #{body[0..200]})"
85
- else
86
- batch_records << { :message_body => body, :delay_seconds => @delay_seconds }
87
- end
88
- end
89
-
90
- until send_batches.length <= 0 do
91
- records = send_batches.shift
92
- until records.length <= 0 do
93
- @queue.batch_send(records.slice!(0..9))
94
- end
95
- end
96
- end
100
+ def generate_id
101
+ @tag_property_name + Time.now.to_i.to_s
97
102
  end
103
+ end
98
104
  end
@@ -1,113 +1,120 @@
1
1
  require 'spec_helper'
2
+ require 'fluent/plugin/in_sqs'
2
3
 
3
- describe do
4
- let(:driver) {
5
- AWS.stub!
4
+ describe Fluent::SQSInput do
5
+ let(:driver) do
6
6
  Fluent::Test.setup
7
7
  Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
8
- }
9
- let(:instance) {driver.instance}
8
+ end
9
+ subject { driver.instance }
10
10
 
11
- describe 'config' do
12
- let(:config) {
13
- %[
11
+ describe '#configure' do
12
+ let(:config) do
13
+ %(
14
14
  aws_key_id AWS_KEY_ID
15
15
  aws_sec_key AWS_SEC_KEY
16
- tag TAG
17
- sqs_url SQS_URL
16
+ tag TAG
17
+ region REGION
18
+ sqs_url http://SQS_URL
19
+ receive_interval 1
18
20
  max_number_of_messages 10
19
21
  wait_time_seconds 10
20
- ]
21
- }
22
-
23
- context do
24
- subject {instance.aws_key_id}
25
- it{should == 'AWS_KEY_ID'}
26
- end
27
-
28
- context do
29
- subject {instance.aws_sec_key}
30
- it{should == 'AWS_SEC_KEY'}
22
+ visibility_timeout 1
23
+ delete_message true
24
+ stub_responses true
25
+ )
31
26
  end
32
27
 
33
- context do
34
- subject {instance.tag}
35
- it{should == 'TAG'}
28
+ context 'fluentd input configuration settings' do
29
+ it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
30
+ it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
31
+ it { expect(subject.tag).to eq('TAG') }
32
+ it { expect(subject.region).to eq('REGION') }
33
+ it { expect(subject.sqs_url).to eq('http://SQS_URL') }
34
+ it { expect(subject.receive_interval).to eq(1) }
35
+ it { expect(subject.max_number_of_messages).to eq(10) }
36
+ it { expect(subject.wait_time_seconds).to eq(10) }
37
+ it { expect(subject.visibility_timeout).to eq(1) }
38
+ it { expect(subject.delete_message).to eq(true) }
39
+ it { expect(subject.stub_responses).to eq(true) }
36
40
  end
37
41
 
38
- context do
39
- subject {instance.sqs_url}
40
- it{should == 'SQS_URL'}
41
- end
42
-
43
- context do
44
- subject {instance.receive_interval}
45
- it{should == 0.1}
46
- end
42
+ context 'AWS configuration settings' do
43
+ subject { Aws.config }
47
44
 
48
- context do
49
- subject {instance.max_number_of_messages}
50
- it{should == 10}
51
- end
45
+ before { driver.instance }
52
46
 
53
- context do
54
- subject {instance.wait_time_seconds}
55
- it{should == 10}
47
+ it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
48
+ it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
49
+ it { expect(subject[:region]).to eq('REGION') }
56
50
  end
57
51
  end
58
-
59
- describe 'emit' do
60
- let(:message) do
61
- { 'body' => 'body',
62
- 'handle' => 'handle',
63
- 'id' => 'id',
64
- 'md5' => 'md5',
65
- 'url' => 'url',
66
- 'sender_id' => 'sender_id'
52
+
53
+ describe '#run' do
54
+ let(:message_attributes) do
55
+ {
56
+ body: 'body',
57
+ receipt_handle: 'receipt_handle',
58
+ message_id: 'message_id',
59
+ md5_of_body: 'md5_of_body',
60
+ queue_url: 'queue_url',
61
+ attributes: { 'SenderId' => 'sender_id' }
67
62
  }
68
63
  end
69
- let(:emmits) {
70
- allow(Time).to receive(:now).and_return(0)
64
+ let(:queue) { double(:queue, receive_messages: true) }
65
+ let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
66
+ let(:messages) { [message] }
71
67
 
72
- class AWS::SQS::Queue
73
- def receive_message(opts)
74
- yield OpenStruct.new(
75
- { 'body' => 'body',
76
- 'handle' => 'handle',
77
- 'id' => 'id',
78
- 'md5' => 'md5',
79
- 'queue' => OpenStruct.new(:url => 'url'),
80
- 'sender_id' => 'sender_id',
81
- 'sent_at' => 0
82
- })
83
- end
68
+ context 'with no delete messages param' do
69
+ let(:config) do
70
+ %(
71
+ tag TAG
72
+ max_number_of_messages 10
73
+ wait_time_seconds 10
74
+ visibility_timeout 1
75
+ delete_message false
76
+ )
84
77
  end
85
- expect_any_instance_of(AWS::SQS::Queue).to receive(:receive_message).with({:limit => 10, :wait_time_seconds=>10}).at_least(:once).and_call_original
86
78
 
87
- d = driver
88
- d.run do
89
- sleep 2
79
+ before do
80
+ allow(subject).to receive(:queue) { queue }
90
81
  end
91
82
 
92
- d.emits
93
- }
94
-
95
- context do
96
- let(:config) {
97
- %[
98
- aws_key_id AWS_KEY_ID
99
- aws_sec_key AWS_SEC_KEY
100
- tag TAG
101
- sqs_url SQS_URL
102
- max_number_of_messages 10
103
- wait_time_seconds 10
104
- ]
105
- }
83
+ it 'parse through messages and emit it' do
84
+ expect(queue).to receive(:receive_messages)
85
+ .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
86
+ expect(subject).to receive(:parse_message).with(message) { message_attributes }
87
+ expect(message).not_to receive(:delete)
88
+ expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
106
89
 
107
- subject {emmits.first}
108
- it{should == ['TAG', 0, message]}
90
+ subject.run
91
+ end
109
92
  end
110
93
 
111
- end
94
+ context 'with no delete messages param' do
95
+ let(:config) do
96
+ %(
97
+ tag TAG
98
+ max_number_of_messages 10
99
+ wait_time_seconds 10
100
+ visibility_timeout 1
101
+ delete_message true
102
+ )
103
+ end
112
104
 
105
+ before do
106
+ allow(subject).to receive(:queue) { queue }
107
+ end
108
+
109
+ it 'parse through messages and emit it' do
110
+ expect(queue).to receive(:receive_messages)
111
+ .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
112
+ expect(subject).to receive(:parse_message).with(message) { message_attributes }
113
+ expect(message).to receive(:delete)
114
+ expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
115
+
116
+ subject.run
117
+ end
118
+ end
119
+ end
113
120
  end
@@ -0,0 +1,129 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/out_sqs'
3
+
4
+ describe Fluent::SQSOutput do
5
+ let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQSOutput) }
6
+ subject { driver.instance }
7
+
8
+ before do
9
+ Fluent::Test.setup
10
+ driver.configure(config)
11
+ end
12
+
13
+ describe '#configure' do
14
+ let(:config) do
15
+ %(
16
+ aws_key_id AWS_KEY_ID
17
+ aws_sec_key AWS_SEC_KEY
18
+ queue_name QUEUE_NAME
19
+ sqs_url http://SQS_URL
20
+ create_queue false
21
+ region REGION
22
+ delay_seconds 1
23
+ include_tag false
24
+ tag_property_name TAG_PROPERTY_NAME
25
+ )
26
+ end
27
+
28
+ context 'fluentd output configuration settings' do
29
+ it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
30
+ it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
31
+ it { expect(subject.queue_name).to eq('QUEUE_NAME') }
32
+ it { expect(subject.sqs_url).to eq('http://SQS_URL') }
33
+ it { expect(subject.create_queue).to eq(false) }
34
+ it { expect(subject.region).to eq('REGION') }
35
+ it { expect(subject.delay_seconds).to eq(1) }
36
+ it { expect(subject.include_tag).to eq(false) }
37
+ it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
38
+ end
39
+
40
+ context 'AWS configuration settings' do
41
+ subject { Aws.config }
42
+
43
+ before { driver.instance }
44
+
45
+ it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
46
+ it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
47
+ it { expect(subject[:region]).to eq('REGION') }
48
+ end
49
+ end
50
+
51
+ describe '#queue' do
52
+ context 'when create_queue and queue_name are set' do
53
+ let(:config) do
54
+ %(
55
+ queue_name QUEUE_NAME
56
+ create_queue true
57
+ )
58
+ end
59
+ let(:resource_instance) { double(:resource_instance) }
60
+ let(:queue) { double(:queue) }
61
+
62
+ before { allow(subject).to receive(:resource) { resource_instance } }
63
+
64
+ it 'calls on create_queue with queue_name' do
65
+ expect(resource_instance).to receive(:create_queue).with(queue_name: 'QUEUE_NAME') { queue }
66
+
67
+ expect(subject.queue).to eq(queue)
68
+ end
69
+ end
70
+
71
+ context 'when create_queue is not set but a sqs_url is' do
72
+ let(:config) do
73
+ %(
74
+ queue_name QUEUE_NAME
75
+ create_queue false
76
+ sqs_url SQS_URL
77
+ )
78
+ end
79
+ let(:resource_instance) { double(:resource_instance) }
80
+ let(:queue) { double(:queue) }
81
+
82
+ before { allow(subject).to receive(:resource) { resource_instance } }
83
+
84
+ it 'gets queue from sqs_url' do
85
+ expect(resource_instance).to receive(:queue).with('SQS_URL') { queue }
86
+
87
+ expect(subject.queue).to eq(queue)
88
+ end
89
+ end
90
+
91
+ context 'when create_queue is not set nor sqs_url' do
92
+ let(:config) do
93
+ %(
94
+ queue_name QUEUE_NAME
95
+ create_queue false
96
+ )
97
+ end
98
+ let(:resource_instance) { double(:resource_instance) }
99
+ let(:queue) { double(:queue) }
100
+
101
+ before { allow(subject).to receive(:resource) { resource_instance } }
102
+
103
+ it 'gets queue from queue_name' do
104
+ expect(resource_instance).to receive(:get_queue_by_name).with(queue_name: 'QUEUE_NAME') { queue }
105
+
106
+ expect(subject.queue).to eq(queue)
107
+ end
108
+ end
109
+ end
110
+
111
+ describe '#write' do
112
+ let(:config) do
113
+ %(
114
+ queue_name QUEUE_NAME
115
+ )
116
+ end
117
+
118
+ let(:record) { {} }
119
+ let(:body) { double(:body, bytesize: 1) }
120
+
121
+ it 'send_messages to queue' do
122
+ allow(Yajl).to receive(:dump).with(record) { body }
123
+
124
+ expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
125
+
126
+ driver.emit(record).run
127
+ end
128
+ end
129
+ end
data/spec/spec_helper.rb CHANGED
@@ -5,7 +5,6 @@
5
5
  #
6
6
  # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
7
7
  RSpec.configure do |config|
8
- config.treat_symbols_as_metadata_keys_with_true_values = true
9
8
  config.run_all_when_everything_filtered = true
10
9
  config.filter_run :focus
11
10
 
@@ -18,7 +17,6 @@ RSpec.configure do |config|
18
17
  require 'fluent/load'
19
18
  require 'fluent/test'
20
19
 
21
- require 'fluent/plugin/in_sqs'
22
20
  require 'ostruct'
23
21
  require 'pry'
24
22
  require 'rr'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.8.0
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuri Odagiri
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-09 00:00:00.000000000 Z
11
+ date: 2016-06-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -31,19 +31,19 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: aws-sdk-v1
34
+ name: aws-sdk
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ">="
37
+ - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: '0'
39
+ version: '2'
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ">="
44
+ - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: '0'
46
+ version: '2'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: yajl-ruby
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -170,6 +170,7 @@ files:
170
170
  - lib/fluent/plugin/in_sqs.rb
171
171
  - lib/fluent/plugin/out_sqs.rb
172
172
  - spec/lib/fluent/plugin/in_sqs_spec.rb
173
+ - spec/lib/fluent/plugin/out_sqs_spec.rb
173
174
  - spec/spec_helper.rb
174
175
  homepage: https://github.com/ixixi/fluent-plugin-sqs
175
176
  licenses:
@@ -191,10 +192,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
191
192
  version: '0'
192
193
  requirements: []
193
194
  rubyforge_project:
194
- rubygems_version: 2.5.2
195
+ rubygems_version: 2.4.1
195
196
  signing_key:
196
197
  specification_version: 3
197
198
  summary: Amazon SQS input/output plugin for Fluent event collector
198
199
  test_files:
199
200
  - spec/lib/fluent/plugin/in_sqs_spec.rb
201
+ - spec/lib/fluent/plugin/out_sqs_spec.rb
200
202
  - spec/spec_helper.rb