fluent-plugin-sqs-tarjei 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/AUTHORS +1 -0
- data/README.rdoc +104 -0
- data/Rakefile +9 -0
- data/VERSION +1 -0
- data/lib/fluent/plugin/in_sqs.rb +80 -0
- data/lib/fluent/plugin/out_sqs.rb +123 -0
- data/spec/lib/fluent/plugin/in_sqs_spec.rb +120 -0
- data/spec/lib/fluent/plugin/out_sqs_spec.rb +169 -0
- data/spec/spec_helper.rb +28 -0
- metadata +201 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 833f5e3505fa6d75487e333c5357d30c10d037920ac79018f0b281e53fc41e99
|
4
|
+
data.tar.gz: a13628e2799a067536a5ff6e0e81dd9cc95483215d537a6dd6fb7292e4c8ce8f
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 88813a04b7b6525d69f19868d5c1f8b981ef48e5991dc8d3459a9a649a27a78511de8371b3530959ebed6daf5441a13585a2b9d36c0733b4887bd2df77fb430c
|
7
|
+
data.tar.gz: 3b54c63ba932c1d32ebe38e8c4236419f6dbd06036a8d3aff8815367efe247f5b69d5fdb85f7a5c01879e25f16d0b570ea5920b9bf5dc08c276426071f038249
|
data/AUTHORS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Yuri Odagiri <ixixizko _at_ gmail.com>
|
data/README.rdoc
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
= Amazon SQS output plugin for Fluent
|
2
|
+
|
3
|
+
== Install
|
4
|
+
$ gem install fluent-plugin-sqs
|
5
|
+
|
6
|
+
== Component
|
7
|
+
|
8
|
+
=== SQSOutput
|
9
|
+
|
10
|
+
Store fluent-event as queue message to Amazon SQS.
|
11
|
+
|
12
|
+
=== SQSInput
|
13
|
+
|
14
|
+
Read events from Amazon SQS.
|
15
|
+
|
16
|
+
== Configuration
|
17
|
+
|
18
|
+
=== SQSOutput
|
19
|
+
|
20
|
+
<match sqs.**>
|
21
|
+
|
22
|
+
type sqs
|
23
|
+
|
24
|
+
# url as https://sqs.us-west-2.amazonaws.com/123456789012/myqueue
|
25
|
+
sqs_url {queue_url}
|
26
|
+
|
27
|
+
# following attribute is required if you don't declare a sqs_url
|
28
|
+
queue_name {queue_instance_key}
|
29
|
+
|
30
|
+
# following attributes are required if you don't use IAM Role nor export credentials to ENV
|
31
|
+
|
32
|
+
aws_key_id {your_aws_key_id}
|
33
|
+
aws_sec_key {your_aws_secret_key}
|
34
|
+
|
35
|
+
# following attributes are optional
|
36
|
+
|
37
|
+
create_queue {boolean}
|
38
|
+
region {your_region}
|
39
|
+
|
40
|
+
# following attributes are required if you use FIFO queue
|
41
|
+
|
42
|
+
message_group_id {string}
|
43
|
+
|
44
|
+
### region list ###
|
45
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
46
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
47
|
+
# US-East (Virginia) : us-east-1
|
48
|
+
# US-West (Oregon) : us-west-2
|
49
|
+
# US-West (N.California) : us-west-1
|
50
|
+
# EU-West (Ireland) : eu-west-1
|
51
|
+
# South America (São Paulo) : sa-east-1
|
52
|
+
|
53
|
+
delay_seconds {delivery_delay_seconds}
|
54
|
+
|
55
|
+
include_tag {boolean}
|
56
|
+
tag_property_name {tag's property name in json}
|
57
|
+
|
58
|
+
|
59
|
+
</match>
|
60
|
+
|
61
|
+
=== SQSInput
|
62
|
+
|
63
|
+
<source>
|
64
|
+
|
65
|
+
type sqs
|
66
|
+
|
67
|
+
# following attribute is required
|
68
|
+
|
69
|
+
sqs_url {queue_url}
|
70
|
+
|
71
|
+
# following attributes are required if you don't use IAM Role nor export credentials to ENV
|
72
|
+
|
73
|
+
aws_key_id {your_aws_key_id}
|
74
|
+
aws_sec_key {your_aws_secret_key}
|
75
|
+
|
76
|
+
# following attributes are optional
|
77
|
+
|
78
|
+
region {your_region}
|
79
|
+
|
80
|
+
### region list ###
|
81
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
82
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
83
|
+
# US-East (Virginia) : us-east-1
|
84
|
+
# US-West (Oregon) : us-west-2
|
85
|
+
# US-West (N.California) : us-west-1
|
86
|
+
# EU-West (Ireland) : eu-west-1
|
87
|
+
# South America (São Paulo) : sa-east-1
|
88
|
+
|
89
|
+
receive_interval {receive_interval_seconds}
|
90
|
+
|
91
|
+
max_number_of_messages {max_number_of_messages}
|
92
|
+
|
93
|
+
wait_time_seconds {receive_message_wait_time_seconds}
|
94
|
+
|
95
|
+
delete_message {delete_message_after_receiving}
|
96
|
+
|
97
|
+
</source>
|
98
|
+
|
99
|
+
== Tool
|
100
|
+
|
101
|
+
== Copyright
|
102
|
+
|
103
|
+
Copyright:: Copyright (c) 2011- Yuri Odagiri
|
104
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
3.0.0
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'fluent/plugin/input'
|
2
|
+
require 'aws-sdk-sqs'
|
3
|
+
|
4
|
+
module Fluent::Plugin
|
5
|
+
class SQSInput < Input
|
6
|
+
Fluent::Plugin.register_input('sqs', self)
|
7
|
+
|
8
|
+
helpers :timer
|
9
|
+
|
10
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
11
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
12
|
+
config_param :tag, :string
|
13
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
14
|
+
config_param :sqs_url, :string, default: nil
|
15
|
+
config_param :receive_interval, :time, default: 0.1
|
16
|
+
config_param :max_number_of_messages, :integer, default: 10
|
17
|
+
config_param :wait_time_seconds, :integer, default: 10
|
18
|
+
config_param :visibility_timeout, :integer, default: nil
|
19
|
+
config_param :delete_message, :bool, default: false
|
20
|
+
config_param :stub_responses, :bool, default: false
|
21
|
+
|
22
|
+
def configure(conf)
|
23
|
+
super
|
24
|
+
|
25
|
+
Aws.config = {
|
26
|
+
access_key_id: @aws_key_id,
|
27
|
+
secret_access_key: @aws_sec_key,
|
28
|
+
region: @region
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
def start
|
33
|
+
super
|
34
|
+
|
35
|
+
timer_execute(:in_sqs_run_periodic_timer, @receive_interval, &method(:run))
|
36
|
+
end
|
37
|
+
|
38
|
+
def client
|
39
|
+
@client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
|
40
|
+
end
|
41
|
+
|
42
|
+
def queue
|
43
|
+
@queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
|
44
|
+
end
|
45
|
+
|
46
|
+
def shutdown
|
47
|
+
super
|
48
|
+
end
|
49
|
+
|
50
|
+
def run
|
51
|
+
queue.receive_messages(
|
52
|
+
max_number_of_messages: @max_number_of_messages,
|
53
|
+
wait_time_seconds: @wait_time_seconds,
|
54
|
+
visibility_timeout: @visibility_timeout
|
55
|
+
).each do |message|
|
56
|
+
record = parse_message(message)
|
57
|
+
|
58
|
+
message.delete if @delete_message
|
59
|
+
|
60
|
+
router.emit(@tag, Fluent::Engine.now, record)
|
61
|
+
end
|
62
|
+
rescue
|
63
|
+
log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
|
64
|
+
log.warn_backtrace $ERROR_INFO.backtrace
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def parse_message(message)
|
70
|
+
{
|
71
|
+
'body' => message.body.to_s,
|
72
|
+
'receipt_handle' => message.receipt_handle.to_s,
|
73
|
+
'message_id' => message.message_id.to_s,
|
74
|
+
'md5_of_body' => message.md5_of_body.to_s,
|
75
|
+
'queue_url' => message.queue_url.to_s,
|
76
|
+
'sender_id' => message.attributes['SenderId'].to_s
|
77
|
+
}
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require 'fluent/plugin/output'
|
2
|
+
require 'aws-sdk-sqs'
|
3
|
+
|
4
|
+
module Fluent::Plugin
|
5
|
+
SQS_BATCH_SEND_MAX_MSGS = 10
|
6
|
+
SQS_BATCH_SEND_MAX_SIZE = 262_144
|
7
|
+
|
8
|
+
class SQSOutput < Output
|
9
|
+
Fluent::Plugin.register_output('sqs', self)
|
10
|
+
|
11
|
+
helpers :compat_parameters, :inject
|
12
|
+
|
13
|
+
DEFAULT_BUFFER_TYPE = "memory"
|
14
|
+
|
15
|
+
config_set_default :include_tag_key, false
|
16
|
+
config_set_default :include_time_key, true
|
17
|
+
|
18
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
19
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
20
|
+
config_param :queue_name, :string, default: nil
|
21
|
+
config_param :sqs_url, :string, default: nil
|
22
|
+
config_param :create_queue, :bool, default: true
|
23
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
24
|
+
config_param :delay_seconds, :integer, default: 0
|
25
|
+
config_param :include_tag, :bool, default: true
|
26
|
+
config_param :tag_property_name, :string, default: '__tag'
|
27
|
+
config_param :message_group_id, :string, default: nil
|
28
|
+
|
29
|
+
config_section :buffer do
|
30
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
31
|
+
end
|
32
|
+
|
33
|
+
def configure(conf)
|
34
|
+
compat_parameters_convert(conf, :buffer, :inject)
|
35
|
+
super
|
36
|
+
|
37
|
+
if (!@queue_name.nil? && @queue_name.end_with?('.fifo')) || (!@sqs_url.nil? && @sqs_url.end_with?('.fifo'))
|
38
|
+
raise Fluent::ConfigError, 'message_group_id parameter is required for FIFO queue' if @message_group_id.nil?
|
39
|
+
end
|
40
|
+
|
41
|
+
Aws.config = {
|
42
|
+
access_key_id: @aws_key_id,
|
43
|
+
secret_access_key: @aws_sec_key,
|
44
|
+
region: @region
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
def client
|
49
|
+
@client ||= Aws::SQS::Client.new
|
50
|
+
end
|
51
|
+
|
52
|
+
def resource
|
53
|
+
@resource ||= Aws::SQS::Resource.new(client: client)
|
54
|
+
end
|
55
|
+
|
56
|
+
def queue
|
57
|
+
return @queue if @queue
|
58
|
+
|
59
|
+
@queue = if @create_queue && @queue_name
|
60
|
+
resource.create_queue(queue_name: @queue_name)
|
61
|
+
else
|
62
|
+
@queue = if @sqs_url
|
63
|
+
resource.queue(@sqs_url)
|
64
|
+
else
|
65
|
+
resource.get_queue_by_name(queue_name: @queue_name)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
@queue
|
70
|
+
end
|
71
|
+
|
72
|
+
def format(tag, time, record)
|
73
|
+
record[@tag_property_name] = tag if @include_tag
|
74
|
+
record = inject_values_to_record(tag, time, record)
|
75
|
+
|
76
|
+
record.to_msgpack
|
77
|
+
end
|
78
|
+
|
79
|
+
def formatted_to_msgpack_binary
|
80
|
+
true
|
81
|
+
end
|
82
|
+
|
83
|
+
def multi_workers_ready?
|
84
|
+
true
|
85
|
+
end
|
86
|
+
|
87
|
+
def write(chunk)
|
88
|
+
batch_records = []
|
89
|
+
batch_size = 0
|
90
|
+
send_batches = [batch_records]
|
91
|
+
|
92
|
+
chunk.msgpack_each do |record|
|
93
|
+
body = Yajl.dump(record)
|
94
|
+
batch_size += body.bytesize
|
95
|
+
|
96
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
|
97
|
+
batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
|
98
|
+
batch_records = []
|
99
|
+
batch_size = body.bytesize
|
100
|
+
send_batches << batch_records
|
101
|
+
end
|
102
|
+
|
103
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE
|
104
|
+
log.warn 'Could not push message to SQS, payload exceeds ' \
|
105
|
+
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
|
106
|
+
"(Truncated message: #{body[0..200]})"
|
107
|
+
else
|
108
|
+
id = "#{@tag_property_name}#{SecureRandom.hex(16)}"
|
109
|
+
batch_record = { id: id, message_body: body, delay_seconds: @delay_seconds }
|
110
|
+
batch_record[:message_group_id] = @message_group_id unless @message_group_id.nil?
|
111
|
+
batch_records << batch_record
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
until send_batches.length <= 0
|
116
|
+
records = send_batches.shift
|
117
|
+
until records.length <= 0
|
118
|
+
queue.send_messages(entries: records.slice!(0..9))
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/in_sqs'
|
3
|
+
|
4
|
+
describe Fluent::Plugin::SQSInput do
|
5
|
+
let(:driver) do
|
6
|
+
Fluent::Test.setup
|
7
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::SQSInput).configure(config)
|
8
|
+
end
|
9
|
+
subject { driver.instance }
|
10
|
+
|
11
|
+
describe '#configure' do
|
12
|
+
let(:config) do
|
13
|
+
%(
|
14
|
+
aws_key_id AWS_KEY_ID
|
15
|
+
aws_sec_key AWS_SEC_KEY
|
16
|
+
tag TAG
|
17
|
+
region REGION
|
18
|
+
sqs_url http://SQS_URL
|
19
|
+
receive_interval 1
|
20
|
+
max_number_of_messages 10
|
21
|
+
wait_time_seconds 10
|
22
|
+
visibility_timeout 1
|
23
|
+
delete_message true
|
24
|
+
stub_responses true
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'fluentd input configuration settings' do
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.tag).to eq('TAG') }
|
32
|
+
it { expect(subject.region).to eq('REGION') }
|
33
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
34
|
+
it { expect(subject.receive_interval).to eq(1) }
|
35
|
+
it { expect(subject.max_number_of_messages).to eq(10) }
|
36
|
+
it { expect(subject.wait_time_seconds).to eq(10) }
|
37
|
+
it { expect(subject.visibility_timeout).to eq(1) }
|
38
|
+
it { expect(subject.delete_message).to eq(true) }
|
39
|
+
it { expect(subject.stub_responses).to eq(true) }
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'AWS configuration settings' do
|
43
|
+
subject { Aws.config }
|
44
|
+
|
45
|
+
before { driver.instance }
|
46
|
+
|
47
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
48
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
49
|
+
it { expect(subject[:region]).to eq('REGION') }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe '#run' do
|
54
|
+
let(:message_attributes) do
|
55
|
+
{
|
56
|
+
body: 'body',
|
57
|
+
receipt_handle: 'receipt_handle',
|
58
|
+
message_id: 'message_id',
|
59
|
+
md5_of_body: 'md5_of_body',
|
60
|
+
queue_url: 'queue_url',
|
61
|
+
attributes: { 'SenderId' => 'sender_id' }
|
62
|
+
}
|
63
|
+
end
|
64
|
+
let(:queue) { double(:queue, receive_messages: true) }
|
65
|
+
let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
|
66
|
+
let(:messages) { [message] }
|
67
|
+
|
68
|
+
context 'with no delete messages param' do
|
69
|
+
let(:config) do
|
70
|
+
%(
|
71
|
+
tag TAG
|
72
|
+
max_number_of_messages 10
|
73
|
+
wait_time_seconds 10
|
74
|
+
visibility_timeout 1
|
75
|
+
delete_message false
|
76
|
+
)
|
77
|
+
end
|
78
|
+
|
79
|
+
before do
|
80
|
+
allow(subject).to receive(:queue) { queue }
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'parse through messages and emit it' do
|
84
|
+
expect(queue).to receive(:receive_messages)
|
85
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
86
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
87
|
+
expect(message).not_to receive(:delete)
|
88
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
|
89
|
+
|
90
|
+
subject.run
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
context 'with no delete messages param' do
|
95
|
+
let(:config) do
|
96
|
+
%(
|
97
|
+
tag TAG
|
98
|
+
max_number_of_messages 10
|
99
|
+
wait_time_seconds 10
|
100
|
+
visibility_timeout 1
|
101
|
+
delete_message true
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
105
|
+
before do
|
106
|
+
allow(subject).to receive(:queue) { queue }
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'parse through messages and emit it' do
|
110
|
+
expect(queue).to receive(:receive_messages)
|
111
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
112
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
113
|
+
expect(message).to receive(:delete)
|
114
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
|
115
|
+
|
116
|
+
subject.run
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/out_sqs'
|
3
|
+
|
4
|
+
describe Fluent::Plugin::SQSOutput do
|
5
|
+
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::SQSOutput) }
|
6
|
+
subject { driver.instance }
|
7
|
+
|
8
|
+
before { Fluent::Test.setup }
|
9
|
+
|
10
|
+
describe '#configure' do
|
11
|
+
let(:config) do
|
12
|
+
%(
|
13
|
+
aws_key_id AWS_KEY_ID
|
14
|
+
aws_sec_key AWS_SEC_KEY
|
15
|
+
queue_name QUEUE_NAME
|
16
|
+
sqs_url http://SQS_URL
|
17
|
+
create_queue false
|
18
|
+
region REGION
|
19
|
+
delay_seconds 1
|
20
|
+
include_tag false
|
21
|
+
tag_property_name TAG_PROPERTY_NAME
|
22
|
+
message_group_id MESSAGE_GROUP_ID
|
23
|
+
)
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'fluentd output configuration settings' do
|
27
|
+
before { driver.configure(config) }
|
28
|
+
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.queue_name).to eq('QUEUE_NAME') }
|
32
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
33
|
+
it { expect(subject.create_queue).to eq(false) }
|
34
|
+
it { expect(subject.region).to eq('REGION') }
|
35
|
+
it { expect(subject.delay_seconds).to eq(1) }
|
36
|
+
it { expect(subject.include_tag).to eq(false) }
|
37
|
+
it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
|
38
|
+
it { expect(subject.message_group_id).to eq('MESSAGE_GROUP_ID') }
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'AWS configuration settings' do
|
42
|
+
subject { Aws.config }
|
43
|
+
|
44
|
+
before do
|
45
|
+
driver.instance
|
46
|
+
driver.configure(config)
|
47
|
+
end
|
48
|
+
|
49
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
50
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
51
|
+
it { expect(subject[:region]).to eq('REGION') }
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'using Standard queue' do
|
55
|
+
let(:config) { %( queue_name QUEUE_NAME ) }
|
56
|
+
|
57
|
+
it 'does not raises error' do
|
58
|
+
expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'using FIFO queue and sets message_group_id' do
|
63
|
+
let(:config) do
|
64
|
+
%(
|
65
|
+
queue_name QUEUE_NAME.fifo
|
66
|
+
message_group_id MESSAGE_GROUP_ID
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'does not raise error' do
|
71
|
+
expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
context 'using FIFO queue and does not set message_group_id' do
|
76
|
+
let(:config) { %( queue_name QUEUE_NAME.fifo ) }
|
77
|
+
|
78
|
+
it 'raises error' do
|
79
|
+
expect { driver.configure(config) }.to raise_error(Fluent::ConfigError)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe '#queue' do
|
85
|
+
before { driver.configure(config) }
|
86
|
+
|
87
|
+
context 'when create_queue and queue_name are set' do
|
88
|
+
let(:config) do
|
89
|
+
%(
|
90
|
+
queue_name QUEUE_NAME
|
91
|
+
create_queue true
|
92
|
+
)
|
93
|
+
end
|
94
|
+
let(:resource_instance) { double(:resource_instance) }
|
95
|
+
let(:queue) { double(:queue) }
|
96
|
+
|
97
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
98
|
+
|
99
|
+
it 'calls on create_queue with queue_name' do
|
100
|
+
expect(resource_instance).to receive(:create_queue).with(queue_name: 'QUEUE_NAME') { queue }
|
101
|
+
|
102
|
+
expect(subject.queue).to eq(queue)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
context 'when create_queue is not set but a sqs_url is' do
|
107
|
+
let(:config) do
|
108
|
+
%(
|
109
|
+
queue_name QUEUE_NAME
|
110
|
+
create_queue false
|
111
|
+
sqs_url SQS_URL
|
112
|
+
)
|
113
|
+
end
|
114
|
+
let(:resource_instance) { double(:resource_instance) }
|
115
|
+
let(:queue) { double(:queue) }
|
116
|
+
|
117
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
118
|
+
|
119
|
+
it 'gets queue from sqs_url' do
|
120
|
+
expect(resource_instance).to receive(:queue).with('SQS_URL') { queue }
|
121
|
+
|
122
|
+
expect(subject.queue).to eq(queue)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context 'when create_queue is not set nor sqs_url' do
|
127
|
+
let(:config) do
|
128
|
+
%(
|
129
|
+
queue_name QUEUE_NAME
|
130
|
+
create_queue false
|
131
|
+
)
|
132
|
+
end
|
133
|
+
let(:resource_instance) { double(:resource_instance) }
|
134
|
+
let(:queue) { double(:queue) }
|
135
|
+
|
136
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
137
|
+
|
138
|
+
it 'gets queue from queue_name' do
|
139
|
+
expect(resource_instance).to receive(:get_queue_by_name).with(queue_name: 'QUEUE_NAME') { queue }
|
140
|
+
|
141
|
+
expect(subject.queue).to eq(queue)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
describe '#write' do
|
147
|
+
before { driver.configure(config) }
|
148
|
+
|
149
|
+
let(:config) do
|
150
|
+
%(
|
151
|
+
queue_name QUEUE_NAME
|
152
|
+
)
|
153
|
+
end
|
154
|
+
|
155
|
+
let(:record) { {} }
|
156
|
+
let(:body) { double(:body, bytesize: 1) }
|
157
|
+
|
158
|
+
it 'send_messages to queue' do
|
159
|
+
allow(Yajl).to receive(:dump).with(record) { body }
|
160
|
+
|
161
|
+
expect(driver.instance).to receive(:queue).twice.and_return("QUEUE_NAME")
|
162
|
+
expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
|
163
|
+
|
164
|
+
driver.run(default_tag: "test") do
|
165
|
+
driver.feed(record)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
4
|
+
# loaded once.
|
5
|
+
#
|
6
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
|
+
RSpec.configure do |config|
|
8
|
+
config.run_all_when_everything_filtered = true
|
9
|
+
config.filter_run :focus
|
10
|
+
|
11
|
+
# Run specs in random order to surface order dependencies. If you find an
|
12
|
+
# order dependency and want to debug it, you can fix the order by providing
|
13
|
+
# the seed, which is printed after each run.
|
14
|
+
# --seed 1234
|
15
|
+
config.order = 'random'
|
16
|
+
|
17
|
+
require 'fluent/load'
|
18
|
+
require 'fluent/test'
|
19
|
+
require 'fluent/test/helpers'
|
20
|
+
require 'fluent/test/driver/output'
|
21
|
+
require 'fluent/test/driver/input'
|
22
|
+
|
23
|
+
require 'ostruct'
|
24
|
+
require 'pry'
|
25
|
+
require 'rr'
|
26
|
+
# prevent Test::Unit's AutoRunner from executing during RSpec's rake task
|
27
|
+
Test::Unit.run = true if defined?(Test::Unit) && Test::Unit.respond_to?(:run=)
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,201 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-sqs-tarjei
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 3.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yuri Odagiri
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-04-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.12.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.12.0
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: aws-sdk-sqs
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '1'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '1'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: yajl-ruby
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '1.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: bundler
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: rake
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: rspec
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: rr
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
name: pry
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
type: :development
|
125
|
+
prerelease: false
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: jeweler
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
type: :development
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ">="
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: test-unit
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - ">="
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: 3.0.0
|
152
|
+
type: :development
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: 3.0.0
|
159
|
+
description:
|
160
|
+
email: ixixizko@gmail.com
|
161
|
+
executables: []
|
162
|
+
extensions: []
|
163
|
+
extra_rdoc_files:
|
164
|
+
- README.rdoc
|
165
|
+
files:
|
166
|
+
- AUTHORS
|
167
|
+
- README.rdoc
|
168
|
+
- Rakefile
|
169
|
+
- VERSION
|
170
|
+
- lib/fluent/plugin/in_sqs.rb
|
171
|
+
- lib/fluent/plugin/out_sqs.rb
|
172
|
+
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
173
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
174
|
+
- spec/spec_helper.rb
|
175
|
+
homepage: https://github.com/ixixi/fluent-plugin-sqs
|
176
|
+
licenses:
|
177
|
+
- Apache-2.0
|
178
|
+
metadata: {}
|
179
|
+
post_install_message:
|
180
|
+
rdoc_options: []
|
181
|
+
require_paths:
|
182
|
+
- lib
|
183
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
189
|
+
requirements:
|
190
|
+
- - ">="
|
191
|
+
- !ruby/object:Gem::Version
|
192
|
+
version: '0'
|
193
|
+
requirements: []
|
194
|
+
rubygems_version: 3.0.3
|
195
|
+
signing_key:
|
196
|
+
specification_version: 3
|
197
|
+
summary: Amazon SQS input/output plugin for Fluent event collector
|
198
|
+
test_files:
|
199
|
+
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
200
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
201
|
+
- spec/spec_helper.rb
|