fluent-plugin-sqs 1.8.0 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.rdoc +20 -18
- data/Rakefile +4 -4
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_sqs.rb +59 -44
- data/lib/fluent/plugin/out_sqs.rb +85 -79
- data/spec/lib/fluent/plugin/in_sqs_spec.rb +91 -84
- data/spec/lib/fluent/plugin/out_sqs_spec.rb +129 -0
- data/spec/spec_helper.rb +0 -2
- metadata +10 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c7d738c77ff00141662a60dbe4db65109c7c873b
|
4
|
+
data.tar.gz: f399c2e1ee216a6f84cc7c172e22950a4a3e8b0c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d55dc8fe951b822e8b047fcf29f8dbf1ab07775292afa0b1d9dc621e82e2c8b489a8397488a894a22ef90c78210d1de9506483e9a92b2878616ac7da50a2717
|
7
|
+
data.tar.gz: af34dacba968a84f73422c7fd0c94bf1eb924466cfdba99d1f3c06030cae047ce8739424c1470b94045f157d8053facc701ad3f17700163896ea5d4d4d1ac318
|
data/README.rdoc
CHANGED
@@ -35,16 +35,16 @@ Read events from from amazon SQS.
|
|
35
35
|
# following attibutes are optional
|
36
36
|
|
37
37
|
create_queue {boolean}
|
38
|
-
|
38
|
+
region {your_region}
|
39
39
|
|
40
|
-
###
|
41
|
-
# Asia Pacific (Tokyo) [Default] :
|
42
|
-
# Asia Pacific (Singapore) :
|
43
|
-
# US-East (Virginia) :
|
44
|
-
# US-West (Oregon) :
|
45
|
-
# US-West (N.California) :
|
46
|
-
# EU-West (Ireland) :
|
47
|
-
# South America (São Paulo) :
|
40
|
+
### region list ###
|
41
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
42
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
43
|
+
# US-East (Virginia) : us-east-1
|
44
|
+
# US-West (Oregon) : us-west-2
|
45
|
+
# US-West (N.California) : us-west-1
|
46
|
+
# EU-West (Ireland) : eu-west-1
|
47
|
+
# South America (São Paulo) : sa-east-1
|
48
48
|
|
49
49
|
delay_seconds {delivery_delay_seconds}
|
50
50
|
|
@@ -71,16 +71,16 @@ Read events from from amazon SQS.
|
|
71
71
|
|
72
72
|
# following attibutes are optional
|
73
73
|
|
74
|
-
|
74
|
+
region {your_region}
|
75
75
|
|
76
|
-
###
|
77
|
-
# Asia Pacific (Tokyo) [Default] :
|
78
|
-
# Asia Pacific (Singapore) :
|
79
|
-
# US-East (Virginia) :
|
80
|
-
# US-West (Oregon) :
|
81
|
-
# US-West (N.California) :
|
82
|
-
# EU-West (Ireland) :
|
83
|
-
# South America (São Paulo) :
|
76
|
+
### region list ###
|
77
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
78
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
79
|
+
# US-East (Virginia) : us-east-1
|
80
|
+
# US-West (Oregon) : us-west-2
|
81
|
+
# US-West (N.California) : us-west-1
|
82
|
+
# EU-West (Ireland) : eu-west-1
|
83
|
+
# South America (São Paulo) : sa-east-1
|
84
84
|
|
85
85
|
receive_interval {receive_interval_seconds}
|
86
86
|
|
@@ -88,6 +88,8 @@ Read events from from amazon SQS.
|
|
88
88
|
|
89
89
|
wait_time_seconds {receive_message_wait_time_seconds}
|
90
90
|
|
91
|
+
delete_message {delete_message_after_receiving}
|
92
|
+
|
91
93
|
</source>
|
92
94
|
|
93
95
|
== Tool
|
data/Rakefile
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'rake'
|
2
2
|
require 'rake/testtask'
|
3
3
|
require 'rake/clean'
|
4
|
-
require
|
5
|
-
require
|
4
|
+
require 'bundler/gem_tasks'
|
5
|
+
require 'rspec/core/rake_task'
|
6
6
|
|
7
|
-
RSpec::Core::RakeTask.new(
|
7
|
+
RSpec::Core::RakeTask.new('spec')
|
8
8
|
|
9
|
-
task :
|
9
|
+
task default: [:build]
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.0.0
|
data/lib/fluent/plugin/in_sqs.rb
CHANGED
@@ -1,46 +1,49 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'aws-sdk-v1'
|
1
|
+
require 'fluent/input'
|
2
|
+
require 'aws-sdk'
|
4
3
|
|
4
|
+
module Fluent
|
5
5
|
class SQSInput < Input
|
6
6
|
Plugin.register_input('sqs', self)
|
7
7
|
|
8
|
-
unless method_defined?(:router)
|
9
|
-
define_method("router") { Fluent::Engine }
|
10
|
-
end
|
8
|
+
define_method('router') { Fluent::Engine } unless method_defined?(:router)
|
11
9
|
|
12
|
-
|
13
|
-
|
14
|
-
end
|
15
|
-
|
16
|
-
config_param :aws_key_id, :string, :default => nil, :secret => true
|
17
|
-
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
10
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
11
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
18
12
|
config_param :tag, :string
|
19
|
-
config_param :
|
20
|
-
config_param :sqs_url, :string
|
21
|
-
config_param :receive_interval, :time, :
|
22
|
-
config_param :max_number_of_messages, :integer, :
|
23
|
-
config_param :wait_time_seconds, :integer, :
|
13
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
14
|
+
config_param :sqs_url, :string, default: nil
|
15
|
+
config_param :receive_interval, :time, default: 0.1
|
16
|
+
config_param :max_number_of_messages, :integer, default: 10
|
17
|
+
config_param :wait_time_seconds, :integer, default: 10
|
18
|
+
config_param :visibility_timeout, :integer, default: nil
|
19
|
+
config_param :delete_message, :bool, default: false
|
20
|
+
config_param :stub_responses, :bool, default: false
|
24
21
|
|
25
22
|
def configure(conf)
|
26
23
|
super
|
27
24
|
|
25
|
+
Aws.config = {
|
26
|
+
access_key_id: @aws_key_id,
|
27
|
+
secret_access_key: @aws_sec_key,
|
28
|
+
region: @region
|
29
|
+
}
|
28
30
|
end
|
29
31
|
|
30
32
|
def start
|
31
33
|
super
|
32
34
|
|
33
|
-
AWS.config(
|
34
|
-
:access_key_id => @aws_key_id,
|
35
|
-
:secret_access_key => @aws_sec_key
|
36
|
-
)
|
37
|
-
|
38
|
-
@queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
|
39
|
-
|
40
35
|
@finished = false
|
41
36
|
@thread = Thread.new(&method(:run_periodic))
|
42
37
|
end
|
43
38
|
|
39
|
+
def client
|
40
|
+
@client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
|
41
|
+
end
|
42
|
+
|
43
|
+
def queue
|
44
|
+
@queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
|
45
|
+
end
|
46
|
+
|
44
47
|
def shutdown
|
45
48
|
super
|
46
49
|
|
@@ -50,27 +53,39 @@ module Fluent
|
|
50
53
|
|
51
54
|
def run_periodic
|
52
55
|
until @finished
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
rescue
|
70
|
-
$log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
|
71
|
-
$log.warn_backtrace $!.backtrace
|
72
|
-
end
|
56
|
+
sleep @receive_interval
|
57
|
+
run
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def run
|
62
|
+
queue.receive_messages(
|
63
|
+
max_number_of_messages: @max_number_of_messages,
|
64
|
+
wait_time_seconds: @wait_time_seconds,
|
65
|
+
visibility_timeout: @visibility_timeout
|
66
|
+
).each do |message|
|
67
|
+
record = parse_message(message)
|
68
|
+
|
69
|
+
message.delete if @delete_message
|
70
|
+
|
71
|
+
router.emit(@tag, Time.now.to_i, record)
|
73
72
|
end
|
73
|
+
rescue
|
74
|
+
$log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
|
75
|
+
$log.warn_backtrace $ERROR_INFO.backtrace
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def parse_message(message)
|
81
|
+
{
|
82
|
+
'body' => message.body.to_s,
|
83
|
+
'receipt_handle' => message.receipt_handle.to_s,
|
84
|
+
'message_id' => message.message_id.to_s,
|
85
|
+
'md5_of_body' => message.md5_of_body.to_s,
|
86
|
+
'queue_url' => message.queue_url.to_s,
|
87
|
+
'sender_id' => message.attributes['SenderId'].to_s
|
88
|
+
}
|
74
89
|
end
|
75
90
|
end
|
76
91
|
end
|
@@ -1,98 +1,104 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'aws-sdk-v1'
|
1
|
+
require 'fluent/output'
|
2
|
+
require 'aws-sdk'
|
4
3
|
|
5
|
-
|
6
|
-
|
4
|
+
module Fluent
|
5
|
+
SQS_BATCH_SEND_MAX_MSGS = 10
|
6
|
+
SQS_BATCH_SEND_MAX_SIZE = 262_144
|
7
|
+
|
8
|
+
class SQSOutput < BufferedOutput
|
9
|
+
Fluent::Plugin.register_output('sqs', self)
|
10
|
+
|
11
|
+
include SetTagKeyMixin
|
12
|
+
config_set_default :include_tag_key, false
|
13
|
+
|
14
|
+
include SetTimeKeyMixin
|
15
|
+
config_set_default :include_time_key, true
|
16
|
+
|
17
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
18
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
19
|
+
config_param :queue_name, :string, default: nil
|
20
|
+
config_param :sqs_url, :string, default: nil
|
21
|
+
config_param :create_queue, :bool, default: true
|
22
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
23
|
+
config_param :delay_seconds, :integer, default: 0
|
24
|
+
config_param :include_tag, :bool, default: true
|
25
|
+
config_param :tag_property_name, :string, default: '__tag'
|
26
|
+
|
27
|
+
def configure(conf)
|
28
|
+
super
|
29
|
+
|
30
|
+
Aws.config = {
|
31
|
+
access_key_id: @aws_key_id,
|
32
|
+
secret_access_key: @aws_sec_key,
|
33
|
+
region: @region
|
34
|
+
}
|
35
|
+
end
|
7
36
|
|
8
|
-
|
9
|
-
|
37
|
+
def client
|
38
|
+
@client ||= Aws::SQS::Client.new
|
39
|
+
end
|
10
40
|
|
11
|
-
|
12
|
-
|
41
|
+
def resource
|
42
|
+
@resource ||= Aws::SQS::Resource.new(client: client)
|
43
|
+
end
|
13
44
|
|
14
|
-
|
15
|
-
|
45
|
+
def queue
|
46
|
+
return @queue if @queue
|
16
47
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
48
|
+
@queue = if @create_queue && @queue_name
|
49
|
+
resource.create_queue(queue_name: @queue_name)
|
50
|
+
else
|
51
|
+
@queue = if @sqs_url
|
52
|
+
resource.queue(@sqs_url)
|
53
|
+
else
|
54
|
+
resource.get_queue_by_name(queue_name: @queue_name)
|
55
|
+
end
|
56
|
+
end
|
26
57
|
|
27
|
-
|
28
|
-
|
29
|
-
end
|
58
|
+
@queue
|
59
|
+
end
|
30
60
|
|
31
|
-
|
32
|
-
|
61
|
+
def format(tag, _time, record)
|
62
|
+
record[@tag_property_name] = tag if @include_tag
|
33
63
|
|
34
|
-
|
35
|
-
|
36
|
-
:secret_access_key => @aws_sec_key)
|
64
|
+
record.to_msgpack
|
65
|
+
end
|
37
66
|
|
38
|
-
|
67
|
+
def write(chunk)
|
68
|
+
batch_records = []
|
69
|
+
batch_size = 0
|
70
|
+
send_batches = [batch_records]
|
39
71
|
|
40
|
-
|
41
|
-
|
72
|
+
chunk.msgpack_each do |record|
|
73
|
+
body = Yajl.dump(record)
|
74
|
+
batch_size += body.bytesize
|
42
75
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
else
|
49
|
-
@queue = @sqs.queues.named(@queue_name)
|
50
|
-
end
|
51
|
-
end
|
76
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
|
77
|
+
batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
|
78
|
+
batch_records = []
|
79
|
+
batch_size = body.bytesize
|
80
|
+
send_batches << batch_records
|
52
81
|
end
|
53
82
|
|
54
|
-
|
55
|
-
|
83
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE
|
84
|
+
log.warn 'Could not push message to SQS, payload exceeds ' \
|
85
|
+
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
|
86
|
+
"(Truncated message: #{body[0..200]})"
|
87
|
+
else
|
88
|
+
batch_records << { id: generate_id, message_body: body, delay_seconds: @delay_seconds }
|
56
89
|
end
|
90
|
+
end
|
57
91
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
record.to_msgpack
|
92
|
+
until send_batches.length <= 0
|
93
|
+
records = send_batches.shift
|
94
|
+
until records.length <= 0
|
95
|
+
queue.send_messages(entries: records.slice!(0..9))
|
64
96
|
end
|
97
|
+
end
|
98
|
+
end
|
65
99
|
|
66
|
-
|
67
|
-
|
68
|
-
batch_size = 0
|
69
|
-
send_batches = [batch_records]
|
70
|
-
|
71
|
-
chunk.msgpack_each do |record|
|
72
|
-
body = Yajl.dump(record)
|
73
|
-
batch_size += body.bytesize
|
74
|
-
if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
|
75
|
-
batch_records.length >= SQS_BATCH_SEND_MAX_MSGS then
|
76
|
-
batch_records = []
|
77
|
-
batch_size = body.bytesize
|
78
|
-
send_batches << batch_records
|
79
|
-
end
|
80
|
-
|
81
|
-
if batch_size > SQS_BATCH_SEND_MAX_SIZE then
|
82
|
-
log.warn "Could not push message to SQS, payload exceeds "\
|
83
|
-
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. "\
|
84
|
-
"(Truncated message: #{body[0..200]})"
|
85
|
-
else
|
86
|
-
batch_records << { :message_body => body, :delay_seconds => @delay_seconds }
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
until send_batches.length <= 0 do
|
91
|
-
records = send_batches.shift
|
92
|
-
until records.length <= 0 do
|
93
|
-
@queue.batch_send(records.slice!(0..9))
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
100
|
+
def generate_id
|
101
|
+
@tag_property_name + Time.now.to_i.to_s
|
97
102
|
end
|
103
|
+
end
|
98
104
|
end
|
@@ -1,113 +1,120 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/in_sqs'
|
2
3
|
|
3
|
-
describe do
|
4
|
-
let(:driver)
|
5
|
-
AWS.stub!
|
4
|
+
describe Fluent::SQSInput do
|
5
|
+
let(:driver) do
|
6
6
|
Fluent::Test.setup
|
7
7
|
Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
|
8
|
-
|
9
|
-
|
8
|
+
end
|
9
|
+
subject { driver.instance }
|
10
10
|
|
11
|
-
describe '
|
12
|
-
let(:config)
|
13
|
-
%
|
11
|
+
describe '#configure' do
|
12
|
+
let(:config) do
|
13
|
+
%(
|
14
14
|
aws_key_id AWS_KEY_ID
|
15
15
|
aws_sec_key AWS_SEC_KEY
|
16
|
-
tag
|
17
|
-
|
16
|
+
tag TAG
|
17
|
+
region REGION
|
18
|
+
sqs_url http://SQS_URL
|
19
|
+
receive_interval 1
|
18
20
|
max_number_of_messages 10
|
19
21
|
wait_time_seconds 10
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
subject {instance.aws_key_id}
|
25
|
-
it{should == 'AWS_KEY_ID'}
|
26
|
-
end
|
27
|
-
|
28
|
-
context do
|
29
|
-
subject {instance.aws_sec_key}
|
30
|
-
it{should == 'AWS_SEC_KEY'}
|
22
|
+
visibility_timeout 1
|
23
|
+
delete_message true
|
24
|
+
stub_responses true
|
25
|
+
)
|
31
26
|
end
|
32
27
|
|
33
|
-
context do
|
34
|
-
|
35
|
-
it{
|
28
|
+
context 'fluentd input configuration settings' do
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.tag).to eq('TAG') }
|
32
|
+
it { expect(subject.region).to eq('REGION') }
|
33
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
34
|
+
it { expect(subject.receive_interval).to eq(1) }
|
35
|
+
it { expect(subject.max_number_of_messages).to eq(10) }
|
36
|
+
it { expect(subject.wait_time_seconds).to eq(10) }
|
37
|
+
it { expect(subject.visibility_timeout).to eq(1) }
|
38
|
+
it { expect(subject.delete_message).to eq(true) }
|
39
|
+
it { expect(subject.stub_responses).to eq(true) }
|
36
40
|
end
|
37
41
|
|
38
|
-
context do
|
39
|
-
subject {
|
40
|
-
it{should == 'SQS_URL'}
|
41
|
-
end
|
42
|
-
|
43
|
-
context do
|
44
|
-
subject {instance.receive_interval}
|
45
|
-
it{should == 0.1}
|
46
|
-
end
|
42
|
+
context 'AWS configuration settings' do
|
43
|
+
subject { Aws.config }
|
47
44
|
|
48
|
-
|
49
|
-
subject {instance.max_number_of_messages}
|
50
|
-
it{should == 10}
|
51
|
-
end
|
45
|
+
before { driver.instance }
|
52
46
|
|
53
|
-
|
54
|
-
|
55
|
-
it{
|
47
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
48
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
49
|
+
it { expect(subject[:region]).to eq('REGION') }
|
56
50
|
end
|
57
51
|
end
|
58
|
-
|
59
|
-
describe '
|
60
|
-
let(:
|
61
|
-
{
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
52
|
+
|
53
|
+
describe '#run' do
|
54
|
+
let(:message_attributes) do
|
55
|
+
{
|
56
|
+
body: 'body',
|
57
|
+
receipt_handle: 'receipt_handle',
|
58
|
+
message_id: 'message_id',
|
59
|
+
md5_of_body: 'md5_of_body',
|
60
|
+
queue_url: 'queue_url',
|
61
|
+
attributes: { 'SenderId' => 'sender_id' }
|
67
62
|
}
|
68
63
|
end
|
69
|
-
let(:
|
70
|
-
|
64
|
+
let(:queue) { double(:queue, receive_messages: true) }
|
65
|
+
let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
|
66
|
+
let(:messages) { [message] }
|
71
67
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
'sent_at' => 0
|
82
|
-
})
|
83
|
-
end
|
68
|
+
context 'with no delete messages param' do
|
69
|
+
let(:config) do
|
70
|
+
%(
|
71
|
+
tag TAG
|
72
|
+
max_number_of_messages 10
|
73
|
+
wait_time_seconds 10
|
74
|
+
visibility_timeout 1
|
75
|
+
delete_message false
|
76
|
+
)
|
84
77
|
end
|
85
|
-
expect_any_instance_of(AWS::SQS::Queue).to receive(:receive_message).with({:limit => 10, :wait_time_seconds=>10}).at_least(:once).and_call_original
|
86
78
|
|
87
|
-
|
88
|
-
|
89
|
-
sleep 2
|
79
|
+
before do
|
80
|
+
allow(subject).to receive(:queue) { queue }
|
90
81
|
end
|
91
82
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
aws_key_id AWS_KEY_ID
|
99
|
-
aws_sec_key AWS_SEC_KEY
|
100
|
-
tag TAG
|
101
|
-
sqs_url SQS_URL
|
102
|
-
max_number_of_messages 10
|
103
|
-
wait_time_seconds 10
|
104
|
-
]
|
105
|
-
}
|
83
|
+
it 'parse through messages and emit it' do
|
84
|
+
expect(queue).to receive(:receive_messages)
|
85
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
86
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
87
|
+
expect(message).not_to receive(:delete)
|
88
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
|
106
89
|
|
107
|
-
|
108
|
-
|
90
|
+
subject.run
|
91
|
+
end
|
109
92
|
end
|
110
93
|
|
111
|
-
|
94
|
+
context 'with no delete messages param' do
|
95
|
+
let(:config) do
|
96
|
+
%(
|
97
|
+
tag TAG
|
98
|
+
max_number_of_messages 10
|
99
|
+
wait_time_seconds 10
|
100
|
+
visibility_timeout 1
|
101
|
+
delete_message true
|
102
|
+
)
|
103
|
+
end
|
112
104
|
|
105
|
+
before do
|
106
|
+
allow(subject).to receive(:queue) { queue }
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'parse through messages and emit it' do
|
110
|
+
expect(queue).to receive(:receive_messages)
|
111
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
112
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
113
|
+
expect(message).to receive(:delete)
|
114
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Integer), message_attributes)
|
115
|
+
|
116
|
+
subject.run
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
113
120
|
end
|
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/out_sqs'
|
3
|
+
|
4
|
+
describe Fluent::SQSOutput do
|
5
|
+
let(:driver) { Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQSOutput) }
|
6
|
+
subject { driver.instance }
|
7
|
+
|
8
|
+
before do
|
9
|
+
Fluent::Test.setup
|
10
|
+
driver.configure(config)
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#configure' do
|
14
|
+
let(:config) do
|
15
|
+
%(
|
16
|
+
aws_key_id AWS_KEY_ID
|
17
|
+
aws_sec_key AWS_SEC_KEY
|
18
|
+
queue_name QUEUE_NAME
|
19
|
+
sqs_url http://SQS_URL
|
20
|
+
create_queue false
|
21
|
+
region REGION
|
22
|
+
delay_seconds 1
|
23
|
+
include_tag false
|
24
|
+
tag_property_name TAG_PROPERTY_NAME
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'fluentd output configuration settings' do
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.queue_name).to eq('QUEUE_NAME') }
|
32
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
33
|
+
it { expect(subject.create_queue).to eq(false) }
|
34
|
+
it { expect(subject.region).to eq('REGION') }
|
35
|
+
it { expect(subject.delay_seconds).to eq(1) }
|
36
|
+
it { expect(subject.include_tag).to eq(false) }
|
37
|
+
it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'AWS configuration settings' do
|
41
|
+
subject { Aws.config }
|
42
|
+
|
43
|
+
before { driver.instance }
|
44
|
+
|
45
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
46
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
47
|
+
it { expect(subject[:region]).to eq('REGION') }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe '#queue' do
|
52
|
+
context 'when create_queue and queue_name are set' do
|
53
|
+
let(:config) do
|
54
|
+
%(
|
55
|
+
queue_name QUEUE_NAME
|
56
|
+
create_queue true
|
57
|
+
)
|
58
|
+
end
|
59
|
+
let(:resource_instance) { double(:resource_instance) }
|
60
|
+
let(:queue) { double(:queue) }
|
61
|
+
|
62
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
63
|
+
|
64
|
+
it 'calls on create_queue with queue_name' do
|
65
|
+
expect(resource_instance).to receive(:create_queue).with(queue_name: 'QUEUE_NAME') { queue }
|
66
|
+
|
67
|
+
expect(subject.queue).to eq(queue)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'when create_queue is not set but a sqs_url is' do
|
72
|
+
let(:config) do
|
73
|
+
%(
|
74
|
+
queue_name QUEUE_NAME
|
75
|
+
create_queue false
|
76
|
+
sqs_url SQS_URL
|
77
|
+
)
|
78
|
+
end
|
79
|
+
let(:resource_instance) { double(:resource_instance) }
|
80
|
+
let(:queue) { double(:queue) }
|
81
|
+
|
82
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
83
|
+
|
84
|
+
it 'gets queue from sqs_url' do
|
85
|
+
expect(resource_instance).to receive(:queue).with('SQS_URL') { queue }
|
86
|
+
|
87
|
+
expect(subject.queue).to eq(queue)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
context 'when create_queue is not set nor sqs_url' do
|
92
|
+
let(:config) do
|
93
|
+
%(
|
94
|
+
queue_name QUEUE_NAME
|
95
|
+
create_queue false
|
96
|
+
)
|
97
|
+
end
|
98
|
+
let(:resource_instance) { double(:resource_instance) }
|
99
|
+
let(:queue) { double(:queue) }
|
100
|
+
|
101
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
102
|
+
|
103
|
+
it 'gets queue from queue_name' do
|
104
|
+
expect(resource_instance).to receive(:get_queue_by_name).with(queue_name: 'QUEUE_NAME') { queue }
|
105
|
+
|
106
|
+
expect(subject.queue).to eq(queue)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
describe '#write' do
|
112
|
+
let(:config) do
|
113
|
+
%(
|
114
|
+
queue_name QUEUE_NAME
|
115
|
+
)
|
116
|
+
end
|
117
|
+
|
118
|
+
let(:record) { {} }
|
119
|
+
let(:body) { double(:body, bytesize: 1) }
|
120
|
+
|
121
|
+
it 'send_messages to queue' do
|
122
|
+
allow(Yajl).to receive(:dump).with(record) { body }
|
123
|
+
|
124
|
+
expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
|
125
|
+
|
126
|
+
driver.emit(record).run
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -5,7 +5,6 @@
|
|
5
5
|
#
|
6
6
|
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
7
|
RSpec.configure do |config|
|
8
|
-
config.treat_symbols_as_metadata_keys_with_true_values = true
|
9
8
|
config.run_all_when_everything_filtered = true
|
10
9
|
config.filter_run :focus
|
11
10
|
|
@@ -18,7 +17,6 @@ RSpec.configure do |config|
|
|
18
17
|
require 'fluent/load'
|
19
18
|
require 'fluent/test'
|
20
19
|
|
21
|
-
require 'fluent/plugin/in_sqs'
|
22
20
|
require 'ostruct'
|
23
21
|
require 'pry'
|
24
22
|
require 'rr'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuri Odagiri
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -31,19 +31,19 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: aws-sdk
|
34
|
+
name: aws-sdk
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - "
|
37
|
+
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '2'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- - "
|
44
|
+
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '2'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: yajl-ruby
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -170,6 +170,7 @@ files:
|
|
170
170
|
- lib/fluent/plugin/in_sqs.rb
|
171
171
|
- lib/fluent/plugin/out_sqs.rb
|
172
172
|
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
173
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
173
174
|
- spec/spec_helper.rb
|
174
175
|
homepage: https://github.com/ixixi/fluent-plugin-sqs
|
175
176
|
licenses:
|
@@ -191,10 +192,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
192
|
version: '0'
|
192
193
|
requirements: []
|
193
194
|
rubyforge_project:
|
194
|
-
rubygems_version: 2.
|
195
|
+
rubygems_version: 2.4.1
|
195
196
|
signing_key:
|
196
197
|
specification_version: 3
|
197
198
|
summary: Amazon SQS input/output plugin for Fluent event collector
|
198
199
|
test_files:
|
199
200
|
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
201
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
200
202
|
- spec/spec_helper.rb
|