fluent-plugin-sqs 1.8.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.rdoc +20 -18
- data/Rakefile +4 -4
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_sqs.rb +59 -44
- data/lib/fluent/plugin/out_sqs.rb +85 -79
- data/spec/lib/fluent/plugin/in_sqs_spec.rb +91 -84
- data/spec/lib/fluent/plugin/out_sqs_spec.rb +129 -0
- data/spec/spec_helper.rb +0 -2
- metadata +10 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c7d738c77ff00141662a60dbe4db65109c7c873b
|
4
|
+
data.tar.gz: f399c2e1ee216a6f84cc7c172e22950a4a3e8b0c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d55dc8fe951b822e8b047fcf29f8dbf1ab07775292afa0b1d9dc621e82e2c8b489a8397488a894a22ef90c78210d1de9506483e9a92b2878616ac7da50a2717
|
7
|
+
data.tar.gz: af34dacba968a84f73422c7fd0c94bf1eb924466cfdba99d1f3c06030cae047ce8739424c1470b94045f157d8053facc701ad3f17700163896ea5d4d4d1ac318
|
data/README.rdoc
CHANGED
@@ -35,16 +35,16 @@ Read events from from amazon SQS.
|
|
35
35
|
# following attibutes are optional
|
36
36
|
|
37
37
|
create_queue {boolean}
|
38
|
-
|
38
|
+
region {your_region}
|
39
39
|
|
40
|
-
###
|
41
|
-
# Asia Pacific (Tokyo) [Default] :
|
42
|
-
# Asia Pacific (Singapore) :
|
43
|
-
# US-East (Virginia) :
|
44
|
-
# US-West (Oregon) :
|
45
|
-
# US-West (N.California) :
|
46
|
-
# EU-West (Ireland) :
|
47
|
-
# South America (São Paulo) :
|
40
|
+
### region list ###
|
41
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
42
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
43
|
+
# US-East (Virginia) : us-east-1
|
44
|
+
# US-West (Oregon) : us-west-2
|
45
|
+
# US-West (N.California) : us-west-1
|
46
|
+
# EU-West (Ireland) : eu-west-1
|
47
|
+
# South America (São Paulo) : sa-east-1
|
48
48
|
|
49
49
|
delay_seconds {delivery_delay_seconds}
|
50
50
|
|
@@ -71,16 +71,16 @@ Read events from from amazon SQS.
|
|
71
71
|
|
72
72
|
# following attibutes are optional
|
73
73
|
|
74
|
-
|
74
|
+
region {your_region}
|
75
75
|
|
76
|
-
###
|
77
|
-
# Asia Pacific (Tokyo) [Default] :
|
78
|
-
# Asia Pacific (Singapore) :
|
79
|
-
# US-East (Virginia) :
|
80
|
-
# US-West (Oregon) :
|
81
|
-
# US-West (N.California) :
|
82
|
-
# EU-West (Ireland) :
|
83
|
-
# South America (São Paulo) :
|
76
|
+
### region list ###
|
77
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
78
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
79
|
+
# US-East (Virginia) : us-east-1
|
80
|
+
# US-West (Oregon) : us-west-2
|
81
|
+
# US-West (N.California) : us-west-1
|
82
|
+
# EU-West (Ireland) : eu-west-1
|
83
|
+
# South America (São Paulo) : sa-east-1
|
84
84
|
|
85
85
|
receive_interval {receive_interval_seconds}
|
86
86
|
|
@@ -88,6 +88,8 @@ Read events from from amazon SQS.
|
|
88
88
|
|
89
89
|
wait_time_seconds {receive_message_wait_time_seconds}
|
90
90
|
|
91
|
+
delete_message {delete_message_after_receiving}
|
92
|
+
|
91
93
|
</source>
|
92
94
|
|
93
95
|
== Tool
|
data/Rakefile
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'rake'
|
2
2
|
require 'rake/testtask'
|
3
3
|
require 'rake/clean'
|
4
|
-
require
|
5
|
-
require
|
4
|
+
require 'bundler/gem_tasks'
|
5
|
+
require 'rspec/core/rake_task'
|
6
6
|
|
7
|
-
RSpec::Core::RakeTask.new(
|
7
|
+
RSpec::Core::RakeTask.new('spec')
|
8
8
|
|
9
|
-
task :
|
9
|
+
task default: [:build]
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.0.0
|
data/lib/fluent/plugin/in_sqs.rb
CHANGED
@@ -1,46 +1,49 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'aws-sdk-v1'
|
1
|
+
require 'fluent/input'
|
2
|
+
require 'aws-sdk'
|
4
3
|
|
4
|
+
module Fluent
|
5
5
|
class SQSInput < Input
|
6
6
|
Plugin.register_input('sqs', self)
|
7
7
|
|
8
|
-
unless method_defined?(:router)
|
9
|
-
define_method("router") { Fluent::Engine }
|
10
|
-
end
|
8
|
+
define_method('router') { Fluent::Engine } unless method_defined?(:router)
|
11
9
|
|
12
|
-
|
13
|
-
|
14
|
-
end
|
15
|
-
|
16
|
-
config_param :aws_key_id, :string, :default => nil, :secret => true
|
17
|
-
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
10
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
11
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
18
12
|
config_param :tag, :string
|
19
|
-
config_param :
|
20
|
-
config_param :sqs_url, :string
|
21
|
-
config_param :receive_interval, :time, :
|
22
|
-
config_param :max_number_of_messages, :integer, :
|
23
|
-
config_param :wait_time_seconds, :integer, :
|
13
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
14
|
+
config_param :sqs_url, :string, default: nil
|
15
|
+
config_param :receive_interval, :time, default: 0.1
|
16
|
+
config_param :max_number_of_messages, :integer, default: 10
|
17
|
+
config_param :wait_time_seconds, :integer, default: 10
|
18
|
+
config_param :visibility_timeout, :integer, default: nil
|
19
|
+
config_param :delete_message, :bool, default: false
|
20
|
+
config_param :stub_responses, :bool, default: false
|
24
21
|
|
25
22
|
def configure(conf)
|
26
23
|
super
|
27
24
|
|
25
|
+
Aws.config = {
|
26
|
+
access_key_id: @aws_key_id,
|
27
|
+
secret_access_key: @aws_sec_key,
|
28
|
+
region: @region
|
29
|
+
}
|
28
30
|
end
|
29
31
|
|
30
32
|
def start
|
31
33
|
super
|
32
34
|
|
33
|
-
AWS.config(
|
34
|
-
:access_key_id => @aws_key_id,
|
35
|
-
:secret_access_key => @aws_sec_key
|
36
|
-
)
|
37
|
-
|
38
|
-
@queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
|
39
|
-
|
40
35
|
@finished = false
|
41
36
|
@thread = Thread.new(&method(:run_periodic))
|
42
37
|
end
|
43
38
|
|
39
|
+
def client
|
40
|
+
@client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
|
41
|
+
end
|
42
|
+
|
43
|
+
def queue
|
44
|
+
@queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
|
45
|
+
end
|
46
|
+
|
44
47
|
def shutdown
|
45
48
|
super
|
46
49
|
|
@@ -50,27 +53,39 @@ module Fluent
|
|
50
53
|
|
51
54
|
def run_periodic
|
52
55
|
until @finished
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
rescue
|
70
|
-
$log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
|
71
|
-
$log.warn_backtrace $!.backtrace
|
72
|
-
end
|
56
|
+
sleep @receive_interval
|
57
|
+
run
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def run
|
62
|
+
queue.receive_messages(
|
63
|
+
max_number_of_messages: @max_number_of_messages,
|
64
|
+
wait_time_seconds: @wait_time_seconds,
|
65
|
+
visibility_timeout: @visibility_timeout
|
66
|
+
).each do |message|
|
67
|
+
record = parse_message(message)
|
68
|
+
|
69
|
+
message.delete if @delete_message
|
70
|
+
|
71
|
+
router.emit(@tag, Time.now.to_i, record)
|
73
72
|
end
|
73
|
+
rescue
|
74
|
+
$log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
|
75
|
+
$log.warn_backtrace $ERROR_INFO.backtrace
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def parse_message(message)
|
81
|
+
{
|
82
|
+
'body' => message.body.to_s,
|
83
|
+
'receipt_handle' => message.receipt_handle.to_s,
|
84
|
+
'message_id' => message.message_id.to_s,
|
85
|
+
'md5_of_body' => message.md5_of_body.to_s,
|
86
|
+
'queue_url' => message.queue_url.to_s,
|
87
|
+
'sender_id' => message.attributes['SenderId'].to_s
|
88
|
+
}
|
74
89
|
end
|
75
90
|
end
|
76
91
|
end
|
@@ -1,98 +1,104 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'aws-sdk-v1'
|
1
|
+
require 'fluent/output'
|
2
|
+
require 'aws-sdk'
|
4
3
|
|
5
|
-
|
6
|
-
|
4
|
+
module Fluent
|
5
|
+
SQS_BATCH_SEND_MAX_MSGS = 10
|
6
|
+
SQS_BATCH_SEND_MAX_SIZE = 262_144
|
7
|
+
|
8
|
+
class SQSOutput < BufferedOutput
|
9
|
+
Fluent::Plugin.register_output('sqs', self)
|
10
|
+
|
11
|
+
include SetTagKeyMixin
|
12
|
+
config_set_default :include_tag_key, false
|
13
|
+
|
14
|
+
include SetTimeKeyMixin
|
15
|
+
config_set_default :include_time_key, true
|
16
|
+
|
17
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
18
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
19
|
+
config_param :queue_name, :string, default: nil
|
20
|
+
config_param :sqs_url, :string, default: nil
|
21
|
+
config_param :create_queue, :bool, default: true
|
22
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
23
|
+
config_param :delay_seconds, :integer, default: 0
|
24
|
+
config_param :include_tag, :bool, default: true
|
25
|
+
config_param :tag_property_name, :string, default: '__tag'
|
26
|
+
|
27
|
+
def configure(conf)
|
28
|
+
super
|
29
|
+
|
30
|
+
Aws.config = {
|
31
|
+
access_key_id: @aws_key_id,
|
32
|
+
secret_access_key: @aws_sec_key,
|
33
|
+
region: @region
|
34
|
+
}
|
35
|
+
end
|
7
36
|
|
8
|
-
|
9
|
-
|
37
|
+
def client
|
38
|
+
@client ||= Aws::SQS::Client.new
|
39
|
+
end
|
10
40
|
|
11
|
-
|
12
|
-
|
41
|
+
def resource
|
42
|
+
@resource ||= Aws::SQS::Resource.new(client: client)
|
43
|
+
end
|
13
44
|
|
14
|
-
|
15
|
-
|
45
|
+
def queue
|
46
|
+
return @queue if @queue
|
16
47
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
48
|
+
@queue = if @create_queue && @queue_name
|
49
|
+
resource.create_queue(queue_name: @queue_name)
|
50
|
+
else
|
51
|
+
@queue = if @sqs_url
|
52
|
+
resource.queue(@sqs_url)
|
53
|
+
else
|
54
|
+
resource.get_queue_by_name(queue_name: @queue_name)
|
55
|
+
end
|
56
|
+
end
|
26
57
|
|
27
|
-
|
28
|
-
|
29
|
-
end
|
58
|
+
@queue
|
59
|
+
end
|
30
60
|
|
31
|
-
|
32
|
-
|
61
|
+
def format(tag, _time, record)
|
62
|
+
record[@tag_property_name] = tag if @include_tag
|
33
63
|
|
34
|
-
|
35
|
-
|
36
|
-
:secret_access_key => @aws_sec_key)
|
64
|
+
record.to_msgpack
|
65
|
+
end
|
37
66
|
|
38
|
-
|
67
|
+
def write(chunk)
|
68
|
+
batch_records = []
|
69
|
+
batch_size = 0
|
70
|
+
send_batches = [batch_records]
|
39
71
|
|
40
|
-
|
41
|
-
|
72
|
+
chunk.msgpack_each do |record|
|
73
|
+
body = Yajl.dump(record)
|
74
|
+
batch_size += body.bytesize
|
42
75
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
else
|
49
|
-
@queue = @sqs.queues.named(@queue_name)
|
50
|
-
end
|
51
|
-
end
|
76
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
|
77
|
+
batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
|
78
|
+
batch_records = []
|
79
|
+
batch_size = body.bytesize
|
80
|
+
send_batches << batch_records
|
52
81
|
end
|
53
82
|
|
54
|
-
|
55
|
-
|
83
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE
|
84
|
+
log.warn 'Could not push message to SQS, payload exceeds ' \
|
85
|
+
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
|
86
|
+
"(Truncated message: #{body[0..200]})"
|
87
|
+
else
|
88
|
+
batch_records << { id: generate_id, message_body: body, delay_seconds: @delay_seconds }
|
56
89
|
end
|
90
|
+
end
|
57
91
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
record.to_msgpack
|
92
|
+
until send_batches.length <= 0
|
93
|
+
records = send_batches.shift
|
94
|
+
until records.length <= 0
|
95
|
+
queue.send_messages(entries: records.slice!(0..9))
|
64
96
|
end
|
97
|
+
end
|
98
|
+
end
|
65
99
|
|
66
|
-
|
67
|
-
|
68
|
-
batch_size = 0
|
69
|
-
send_batches = [batch_records]
|
70
|
-
|
71
|
-
chunk.msgpack_each do |record|
|
72
|
-
body = Yajl.dump(record)
|
73
|
-
batch_size += body.bytesize
|
74
|
-
if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
|
75
|
-
batch_records.length >= SQS_BATCH_SEND_MAX_MSGS then
|
76
|
-
batch_records = []
|
77
|
-
batch_size = body.bytesize
|
78
|
-
send_batches << batch_records
|
79
|
-
end
|
80
|
-
|
81
|
-
if batch_size > SQS_BATCH_SEND_MAX_SIZE then
|
82
|
-
log.warn "Could not push message to SQS, payload exceeds "\
|
83
|
-
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. "\
|
84
|
-
"(Truncated message: #{body[0..200]})"
|
85
|
-
else
|
86
|
-
batch_records << { :message_body => body, :delay_seconds => @delay_seconds }
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
until send_batches.length <= 0 do
|
91
|
-
records = send_batches.shift
|
92
|
-
until records.length <= 0 do
|
93
|
-
@queue.batch_send(records.slice!(0..9))
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
100
|
+
def generate_id
|
101
|
+
@tag_property_name + Time.now.to_i.to_s
|
97
102
|
end
|
103
|
+
end
|
98
104
|
end
|
@@ -1,113 +1,120 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/in_sqs'
|
2
3
|
|
3
|
-
describe do
|
4
|
-
let(:driver)
|
5
|
-
AWS.stub!
|
4
|
+
describe Fluent::SQSInput do
|
5
|
+
let(:driver) do
|
6
6
|
Fluent::Test.setup
|
7
7
|
Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
|
8
|
-
|
9
|
-
|
8
|
+
end
|
9
|
+
subject { driver.instance }
|
10
10
|
|
11
|
-
describe '
|
12
|
-
let(:config)
|
13
|
-
%
|
11
|
+
describe '#configure' do
|
12
|
+
let(:config) do
|
13
|
+
%(
|
14
14
|
aws_key_id AWS_KEY_ID
|
15
15
|
aws_sec_key AWS_SEC_KEY
|
16
|
-
tag
|
17
|
-
|
16
|
+
tag TAG
|
17
|
+
region REGION
|
18
|
+
sqs_url http://SQS_URL
|
19
|
+
receive_interval 1
|
18
20
|
max_number_of_messages 10
|
19
21
|
wait_time_seconds 10
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
subject {instance.aws_key_id}
|
25
|
-
it{should == 'AWS_KEY_ID'}
|
26
|
-
end
|
27
|
-
|
28
|
-
context do
|
29
|
-
subject {instance.aws_sec_key}
|
30
|
-
it{should == 'AWS_SEC_KEY'}
|
22
|
+
visibility_timeout 1
|
23
|
+
delete_message true
|
24
|
+
stub_responses true
|
25
|
+
)
|
31
26
|
end
|
32
27
|
|
33
|
-
context do
|
34
|
-
|
35
|
-
it{
|
28
|
+
context 'fluentd input configuration settings' do
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.tag).to eq('TAG') }
|
32
|
+
it { expect(subject.region).to eq('REGION') }
|
33
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
34
|
+
it { expect(subject.receive_interval).to eq(1) }
|
35
|
+
it { expect(subject.max_number_of_messages).to eq(10) }
|
36
|
+
it { expect(subject.wait_time_seconds).to eq(10) }
|
37
|
+
it { expect(subject.visibility_timeout).to eq(1) }
|
38
|
+
it { expect(subject.delete_message).to eq(true) }
|
39
|
+
it { expect(subject.stub_responses).to eq(true) }
|
36
40
|
end
|
37
41
|
|
38
|
-
context do
|
39
|
-
subject {
|
40
|
-
it{should == 'SQS_URL'}
|
41
|
-
end
|
42
|
-
|
43
|
-
context do
|
44
|
-
subject {instance.receive_interval}
|
45
|
-
it{should == 0.1}
|
46
|
-
end
|
42
|
+
context 'AWS configuration settings' do
|
43
|
+
subject { Aws.config }
|
47
44
|
|
48
|
-
|
49
|
-
subject {instance.max_number_of_messages}
|
50
|
-
it{should == 10}
|
51
|
-
end
|
45
|
+
before { driver.instance }
|
52
46
|
|
53
|
-
|
54
|
-
|
55
|
-
it{
|
47
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
48
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
49
|
+
it { expect(subject[:region]).to eq('REGION') }
|
56
50
|
end
|
57
51
|
end
|
58
|
-
|
59
|
-
describe '
|
60
|
-
let(:
|
61
|
-
{
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
52
|
+
|
53
|
+
describe '#run' do
|
54
|
+
let(:message_attributes) do
|
55
|
+
{
|
56
|
+
body: 'body',
|
57
|
+
receipt_handle: 'receipt_handle',
|
58
|
+
message_id: 'message_id',
|
59
|
+
md5_of_body: 'md5_of_body',
|
60
|
+
queue_url: 'queue_url',
|
61
|
+
attributes: { 'SenderId' => 'sender_id' }
|
67
62
|
}
|
68
63
|
end
|
69
|
-
let(:
|
70
|
-
|
64
|
+
let(:queue) { double(:queue, receive_messages: true) }
|
65
|
+
let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
|
66
|
+
let(:messages) { [message] }
|
71
67
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
'sent_at' => 0
|
82
|
-
})
|
83
|
-
end
|
68
|
+
context 'with no delete messages param' do
|
69
|
+
let(:config) do
|
70
|
+
%(
|
71
|
+
tag TAG
|
72
|
+
max_number_of_messages 10
|
73
|
+
wait_time_seconds 10
|
74
|
+
visibility_timeout 1
|
75
|
+
delete_message false
|
76
|
+
)
|
84
77
|
end
|
85
|
-
expect_any_instance_of(AWS::SQS::Queue).to receive(:receive_message).with({:limit => 10, :wait_time_seconds=>10}).at_least(:once).and_call_original
|
86
78
|
|
87
|
-
|
88
|
-
|
89
|
-
sleep 2
|
79
|
+
before do
|
80
|
+
allow(subject).to receive(:queue) { queue }
|
90
81
|
end
|
91
82
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
aws_key_id AWS_KEY_ID
|
99
|
-
aws_sec_key AWS_SEC_KEY
|
100
|
-
tag TAG
|
101
|
-
sqs_url SQS_URL
|
102
|
-
max_number_of_messages 10
|
103
|
-
wait_time_seconds 10
|
104
|
-
]
|
105
|
-
}
|
83
|
+
it 'parse through messages and emit it' do
|
84
|
+
expect(queue).to receive(:receive_messages)
|
85
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
86
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
87
|
+
expect(message).not_to receive(:delete)
|
88
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
|
106
89
|
|
107
|
-
|
108
|
-
|
90
|
+
subject.run
|
91
|
+
end
|
109
92
|
end
|
110
93
|
|
111
|
-
|
94
|
+
context 'with no delete messages param' do
|
95
|
+
let(:config) do
|
96
|
+
%(
|
97
|
+
tag TAG
|
98
|
+
max_number_of_messages 10
|
99
|
+
wait_time_seconds 10
|
100
|
+
visibility_timeout 1
|
101
|
+
delete_message true
|
102
|
+
)
|
103
|
+
end
|
112
104
|
|
105
|
+
before do
|
106
|
+
allow(subject).to receive(:queue) { queue }
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'parse through messages and emit it' do
|
110
|
+
expect(queue).to receive(:receive_messages)
|
111
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
112
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
113
|
+
expect(message).to receive(:delete)
|
114
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
|
115
|
+
|
116
|
+
subject.run
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
113
120
|
end
|
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/out_sqs'
|
3
|
+
|
4
|
+
describe Fluent::SQSOutput do
|
5
|
+
let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQSOutput) }
|
6
|
+
subject { driver.instance }
|
7
|
+
|
8
|
+
before do
|
9
|
+
Fluent::Test.setup
|
10
|
+
driver.configure(config)
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#configure' do
|
14
|
+
let(:config) do
|
15
|
+
%(
|
16
|
+
aws_key_id AWS_KEY_ID
|
17
|
+
aws_sec_key AWS_SEC_KEY
|
18
|
+
queue_name QUEUE_NAME
|
19
|
+
sqs_url http://SQS_URL
|
20
|
+
create_queue false
|
21
|
+
region REGION
|
22
|
+
delay_seconds 1
|
23
|
+
include_tag false
|
24
|
+
tag_property_name TAG_PROPERTY_NAME
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'fluentd output configuration settings' do
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.queue_name).to eq('QUEUE_NAME') }
|
32
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
33
|
+
it { expect(subject.create_queue).to eq(false) }
|
34
|
+
it { expect(subject.region).to eq('REGION') }
|
35
|
+
it { expect(subject.delay_seconds).to eq(1) }
|
36
|
+
it { expect(subject.include_tag).to eq(false) }
|
37
|
+
it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'AWS configuration settings' do
|
41
|
+
subject { Aws.config }
|
42
|
+
|
43
|
+
before { driver.instance }
|
44
|
+
|
45
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
46
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
47
|
+
it { expect(subject[:region]).to eq('REGION') }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe '#queue' do
|
52
|
+
context 'when create_queue and queue_name are set' do
|
53
|
+
let(:config) do
|
54
|
+
%(
|
55
|
+
queue_name QUEUE_NAME
|
56
|
+
create_queue true
|
57
|
+
)
|
58
|
+
end
|
59
|
+
let(:resource_instance) { double(:resource_instance) }
|
60
|
+
let(:queue) { double(:queue) }
|
61
|
+
|
62
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
63
|
+
|
64
|
+
it 'calls on create_queue with queue_name' do
|
65
|
+
expect(resource_instance).to receive(:create_queue).with(queue_name: 'QUEUE_NAME') { queue }
|
66
|
+
|
67
|
+
expect(subject.queue).to eq(queue)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'when create_queue is not set but a sqs_url is' do
|
72
|
+
let(:config) do
|
73
|
+
%(
|
74
|
+
queue_name QUEUE_NAME
|
75
|
+
create_queue false
|
76
|
+
sqs_url SQS_URL
|
77
|
+
)
|
78
|
+
end
|
79
|
+
let(:resource_instance) { double(:resource_instance) }
|
80
|
+
let(:queue) { double(:queue) }
|
81
|
+
|
82
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
83
|
+
|
84
|
+
it 'gets queue from sqs_url' do
|
85
|
+
expect(resource_instance).to receive(:queue).with('SQS_URL') { queue }
|
86
|
+
|
87
|
+
expect(subject.queue).to eq(queue)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
context 'when create_queue is not set nor sqs_url' do
|
92
|
+
let(:config) do
|
93
|
+
%(
|
94
|
+
queue_name QUEUE_NAME
|
95
|
+
create_queue false
|
96
|
+
)
|
97
|
+
end
|
98
|
+
let(:resource_instance) { double(:resource_instance) }
|
99
|
+
let(:queue) { double(:queue) }
|
100
|
+
|
101
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
102
|
+
|
103
|
+
it 'gets queue from queue_name' do
|
104
|
+
expect(resource_instance).to receive(:get_queue_by_name).with(queue_name: 'QUEUE_NAME') { queue }
|
105
|
+
|
106
|
+
expect(subject.queue).to eq(queue)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
describe '#write' do
|
112
|
+
let(:config) do
|
113
|
+
%(
|
114
|
+
queue_name QUEUE_NAME
|
115
|
+
)
|
116
|
+
end
|
117
|
+
|
118
|
+
let(:record) { {} }
|
119
|
+
let(:body) { double(:body, bytesize: 1) }
|
120
|
+
|
121
|
+
it 'send_messages to queue' do
|
122
|
+
allow(Yajl).to receive(:dump).with(record) { body }
|
123
|
+
|
124
|
+
expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
|
125
|
+
|
126
|
+
driver.emit(record).run
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -5,7 +5,6 @@
|
|
5
5
|
#
|
6
6
|
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
7
|
RSpec.configure do |config|
|
8
|
-
config.treat_symbols_as_metadata_keys_with_true_values = true
|
9
8
|
config.run_all_when_everything_filtered = true
|
10
9
|
config.filter_run :focus
|
11
10
|
|
@@ -18,7 +17,6 @@ RSpec.configure do |config|
|
|
18
17
|
require 'fluent/load'
|
19
18
|
require 'fluent/test'
|
20
19
|
|
21
|
-
require 'fluent/plugin/in_sqs'
|
22
20
|
require 'ostruct'
|
23
21
|
require 'pry'
|
24
22
|
require 'rr'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuri Odagiri
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -31,19 +31,19 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: aws-sdk
|
34
|
+
name: aws-sdk
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - "
|
37
|
+
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '2'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- - "
|
44
|
+
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '2'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: yajl-ruby
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -170,6 +170,7 @@ files:
|
|
170
170
|
- lib/fluent/plugin/in_sqs.rb
|
171
171
|
- lib/fluent/plugin/out_sqs.rb
|
172
172
|
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
173
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
173
174
|
- spec/spec_helper.rb
|
174
175
|
homepage: https://github.com/ixixi/fluent-plugin-sqs
|
175
176
|
licenses:
|
@@ -191,10 +192,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
192
|
version: '0'
|
192
193
|
requirements: []
|
193
194
|
rubyforge_project:
|
194
|
-
rubygems_version: 2.
|
195
|
+
rubygems_version: 2.4.1
|
195
196
|
signing_key:
|
196
197
|
specification_version: 3
|
197
198
|
summary: Amazon SQS input/output plugin for Fluent event collector
|
198
199
|
test_files:
|
199
200
|
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
201
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
200
202
|
- spec/spec_helper.rb
|