fluent-plugin-sqs-tarjei 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AUTHORS +1 -0
- data/README.rdoc +104 -0
- data/Rakefile +9 -0
- data/VERSION +1 -0
- data/lib/fluent/plugin/in_sqs.rb +80 -0
- data/lib/fluent/plugin/out_sqs.rb +123 -0
- data/spec/lib/fluent/plugin/in_sqs_spec.rb +120 -0
- data/spec/lib/fluent/plugin/out_sqs_spec.rb +169 -0
- data/spec/spec_helper.rb +28 -0
- metadata +201 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 833f5e3505fa6d75487e333c5357d30c10d037920ac79018f0b281e53fc41e99
|
4
|
+
data.tar.gz: a13628e2799a067536a5ff6e0e81dd9cc95483215d537a6dd6fb7292e4c8ce8f
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 88813a04b7b6525d69f19868d5c1f8b981ef48e5991dc8d3459a9a649a27a78511de8371b3530959ebed6daf5441a13585a2b9d36c0733b4887bd2df77fb430c
|
7
|
+
data.tar.gz: 3b54c63ba932c1d32ebe38e8c4236419f6dbd06036a8d3aff8815367efe247f5b69d5fdb85f7a5c01879e25f16d0b570ea5920b9bf5dc08c276426071f038249
|
data/AUTHORS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Yuri Odagiri <ixixizko _at_ gmail.com>
|
data/README.rdoc
ADDED
@@ -0,0 +1,104 @@
|
|
1
|
+
= Amazon SQS output plugin for Fluent
|
2
|
+
|
3
|
+
== Install
|
4
|
+
$ gem install fluent-plugin-sqs
|
5
|
+
|
6
|
+
== Component
|
7
|
+
|
8
|
+
=== SQSOutput
|
9
|
+
|
10
|
+
Store fluent-event as queue message to Amazon SQS.
|
11
|
+
|
12
|
+
=== SQSInput
|
13
|
+
|
14
|
+
Read events from Amazon SQS.
|
15
|
+
|
16
|
+
== Configuration
|
17
|
+
|
18
|
+
=== SQSOutput
|
19
|
+
|
20
|
+
<match sqs.**>
|
21
|
+
|
22
|
+
type sqs
|
23
|
+
|
24
|
+
# url as https://sqs.us-west-2.amazonaws.com/123456789012/myqueue
|
25
|
+
sqs_url {queue_url}
|
26
|
+
|
27
|
+
# following attribute is required if you don't declare a sqs_url
|
28
|
+
queue_name {queue_instance_key}
|
29
|
+
|
30
|
+
# following attributes are required if you don't use IAM Role nor export credentials to ENV
|
31
|
+
|
32
|
+
aws_key_id {your_aws_key_id}
|
33
|
+
aws_sec_key {your_aws_secret_key}
|
34
|
+
|
35
|
+
# following attributes are optional
|
36
|
+
|
37
|
+
create_queue {boolean}
|
38
|
+
region {your_region}
|
39
|
+
|
40
|
+
# following attributes are required if you use FIFO queue
|
41
|
+
|
42
|
+
message_group_id {string}
|
43
|
+
|
44
|
+
### region list ###
|
45
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
46
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
47
|
+
# US-East (Virginia) : us-east-1
|
48
|
+
# US-West (Oregon) : us-west-2
|
49
|
+
# US-West (N.California) : us-west-1
|
50
|
+
# EU-West (Ireland) : eu-west-1
|
51
|
+
# South America (São Paulo) : sa-east-1
|
52
|
+
|
53
|
+
delay_seconds {delivery_delay_seconds}
|
54
|
+
|
55
|
+
include_tag {boolean}
|
56
|
+
tag_property_name {tag's property name in json}
|
57
|
+
|
58
|
+
|
59
|
+
</match>
|
60
|
+
|
61
|
+
=== SQSInput
|
62
|
+
|
63
|
+
<source>
|
64
|
+
|
65
|
+
type sqs
|
66
|
+
|
67
|
+
# following attribute is required
|
68
|
+
|
69
|
+
sqs_url {queue_url}
|
70
|
+
|
71
|
+
# following attributes are required if you don't use IAM Role nor export credentials to ENV
|
72
|
+
|
73
|
+
aws_key_id {your_aws_key_id}
|
74
|
+
aws_sec_key {your_aws_secret_key}
|
75
|
+
|
76
|
+
# following attributes are optional
|
77
|
+
|
78
|
+
region {your_region}
|
79
|
+
|
80
|
+
### region list ###
|
81
|
+
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
82
|
+
# Asia Pacific (Singapore) : ap-southeast-1
|
83
|
+
# US-East (Virginia) : us-east-1
|
84
|
+
# US-West (Oregon) : us-west-2
|
85
|
+
# US-West (N.California) : us-west-1
|
86
|
+
# EU-West (Ireland) : eu-west-1
|
87
|
+
# South America (São Paulo) : sa-east-1
|
88
|
+
|
89
|
+
receive_interval {receive_interval_seconds}
|
90
|
+
|
91
|
+
max_number_of_messages {max_number_of_messages}
|
92
|
+
|
93
|
+
wait_time_seconds {receive_message_wait_time_seconds}
|
94
|
+
|
95
|
+
delete_message {delete_message_after_receiving}
|
96
|
+
|
97
|
+
</source>
|
98
|
+
|
99
|
+
== Tool
|
100
|
+
|
101
|
+
== Copyright
|
102
|
+
|
103
|
+
Copyright:: Copyright (c) 2011- Yuri Odagiri
|
104
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
3.0.0
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'fluent/plugin/input'
|
2
|
+
require 'aws-sdk-sqs'
|
3
|
+
|
4
|
+
module Fluent::Plugin
|
5
|
+
class SQSInput < Input
|
6
|
+
Fluent::Plugin.register_input('sqs', self)
|
7
|
+
|
8
|
+
helpers :timer
|
9
|
+
|
10
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
11
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
12
|
+
config_param :tag, :string
|
13
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
14
|
+
config_param :sqs_url, :string, default: nil
|
15
|
+
config_param :receive_interval, :time, default: 0.1
|
16
|
+
config_param :max_number_of_messages, :integer, default: 10
|
17
|
+
config_param :wait_time_seconds, :integer, default: 10
|
18
|
+
config_param :visibility_timeout, :integer, default: nil
|
19
|
+
config_param :delete_message, :bool, default: false
|
20
|
+
config_param :stub_responses, :bool, default: false
|
21
|
+
|
22
|
+
def configure(conf)
|
23
|
+
super
|
24
|
+
|
25
|
+
Aws.config = {
|
26
|
+
access_key_id: @aws_key_id,
|
27
|
+
secret_access_key: @aws_sec_key,
|
28
|
+
region: @region
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
def start
|
33
|
+
super
|
34
|
+
|
35
|
+
timer_execute(:in_sqs_run_periodic_timer, @receive_interval, &method(:run))
|
36
|
+
end
|
37
|
+
|
38
|
+
def client
|
39
|
+
@client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
|
40
|
+
end
|
41
|
+
|
42
|
+
def queue
|
43
|
+
@queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
|
44
|
+
end
|
45
|
+
|
46
|
+
def shutdown
|
47
|
+
super
|
48
|
+
end
|
49
|
+
|
50
|
+
def run
|
51
|
+
queue.receive_messages(
|
52
|
+
max_number_of_messages: @max_number_of_messages,
|
53
|
+
wait_time_seconds: @wait_time_seconds,
|
54
|
+
visibility_timeout: @visibility_timeout
|
55
|
+
).each do |message|
|
56
|
+
record = parse_message(message)
|
57
|
+
|
58
|
+
message.delete if @delete_message
|
59
|
+
|
60
|
+
router.emit(@tag, Fluent::Engine.now, record)
|
61
|
+
end
|
62
|
+
rescue
|
63
|
+
log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
|
64
|
+
log.warn_backtrace $ERROR_INFO.backtrace
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def parse_message(message)
|
70
|
+
{
|
71
|
+
'body' => message.body.to_s,
|
72
|
+
'receipt_handle' => message.receipt_handle.to_s,
|
73
|
+
'message_id' => message.message_id.to_s,
|
74
|
+
'md5_of_body' => message.md5_of_body.to_s,
|
75
|
+
'queue_url' => message.queue_url.to_s,
|
76
|
+
'sender_id' => message.attributes['SenderId'].to_s
|
77
|
+
}
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require 'fluent/plugin/output'
|
2
|
+
require 'aws-sdk-sqs'
|
3
|
+
|
4
|
+
module Fluent::Plugin
|
5
|
+
SQS_BATCH_SEND_MAX_MSGS = 10
|
6
|
+
SQS_BATCH_SEND_MAX_SIZE = 262_144
|
7
|
+
|
8
|
+
class SQSOutput < Output
|
9
|
+
Fluent::Plugin.register_output('sqs', self)
|
10
|
+
|
11
|
+
helpers :compat_parameters, :inject
|
12
|
+
|
13
|
+
DEFAULT_BUFFER_TYPE = "memory"
|
14
|
+
|
15
|
+
config_set_default :include_tag_key, false
|
16
|
+
config_set_default :include_time_key, true
|
17
|
+
|
18
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
19
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
20
|
+
config_param :queue_name, :string, default: nil
|
21
|
+
config_param :sqs_url, :string, default: nil
|
22
|
+
config_param :create_queue, :bool, default: true
|
23
|
+
config_param :region, :string, default: 'ap-northeast-1'
|
24
|
+
config_param :delay_seconds, :integer, default: 0
|
25
|
+
config_param :include_tag, :bool, default: true
|
26
|
+
config_param :tag_property_name, :string, default: '__tag'
|
27
|
+
config_param :message_group_id, :string, default: nil
|
28
|
+
|
29
|
+
config_section :buffer do
|
30
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
31
|
+
end
|
32
|
+
|
33
|
+
def configure(conf)
|
34
|
+
compat_parameters_convert(conf, :buffer, :inject)
|
35
|
+
super
|
36
|
+
|
37
|
+
if (!@queue_name.nil? && @queue_name.end_with?('.fifo')) || (!@sqs_url.nil? && @sqs_url.end_with?('.fifo'))
|
38
|
+
raise Fluent::ConfigError, 'message_group_id parameter is required for FIFO queue' if @message_group_id.nil?
|
39
|
+
end
|
40
|
+
|
41
|
+
Aws.config = {
|
42
|
+
access_key_id: @aws_key_id,
|
43
|
+
secret_access_key: @aws_sec_key,
|
44
|
+
region: @region
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
def client
|
49
|
+
@client ||= Aws::SQS::Client.new
|
50
|
+
end
|
51
|
+
|
52
|
+
def resource
|
53
|
+
@resource ||= Aws::SQS::Resource.new(client: client)
|
54
|
+
end
|
55
|
+
|
56
|
+
def queue
|
57
|
+
return @queue if @queue
|
58
|
+
|
59
|
+
@queue = if @create_queue && @queue_name
|
60
|
+
resource.create_queue(queue_name: @queue_name)
|
61
|
+
else
|
62
|
+
@queue = if @sqs_url
|
63
|
+
resource.queue(@sqs_url)
|
64
|
+
else
|
65
|
+
resource.get_queue_by_name(queue_name: @queue_name)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
@queue
|
70
|
+
end
|
71
|
+
|
72
|
+
def format(tag, time, record)
|
73
|
+
record[@tag_property_name] = tag if @include_tag
|
74
|
+
record = inject_values_to_record(tag, time, record)
|
75
|
+
|
76
|
+
record.to_msgpack
|
77
|
+
end
|
78
|
+
|
79
|
+
def formatted_to_msgpack_binary
|
80
|
+
true
|
81
|
+
end
|
82
|
+
|
83
|
+
def multi_workers_ready?
|
84
|
+
true
|
85
|
+
end
|
86
|
+
|
87
|
+
def write(chunk)
|
88
|
+
batch_records = []
|
89
|
+
batch_size = 0
|
90
|
+
send_batches = [batch_records]
|
91
|
+
|
92
|
+
chunk.msgpack_each do |record|
|
93
|
+
body = Yajl.dump(record)
|
94
|
+
batch_size += body.bytesize
|
95
|
+
|
96
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
|
97
|
+
batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
|
98
|
+
batch_records = []
|
99
|
+
batch_size = body.bytesize
|
100
|
+
send_batches << batch_records
|
101
|
+
end
|
102
|
+
|
103
|
+
if batch_size > SQS_BATCH_SEND_MAX_SIZE
|
104
|
+
log.warn 'Could not push message to SQS, payload exceeds ' \
|
105
|
+
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
|
106
|
+
"(Truncated message: #{body[0..200]})"
|
107
|
+
else
|
108
|
+
id = "#{@tag_property_name}#{SecureRandom.hex(16)}"
|
109
|
+
batch_record = { id: id, message_body: body, delay_seconds: @delay_seconds }
|
110
|
+
batch_record[:message_group_id] = @message_group_id unless @message_group_id.nil?
|
111
|
+
batch_records << batch_record
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
until send_batches.length <= 0
|
116
|
+
records = send_batches.shift
|
117
|
+
until records.length <= 0
|
118
|
+
queue.send_messages(entries: records.slice!(0..9))
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/in_sqs'
|
3
|
+
|
4
|
+
describe Fluent::Plugin::SQSInput do
|
5
|
+
let(:driver) do
|
6
|
+
Fluent::Test.setup
|
7
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::SQSInput).configure(config)
|
8
|
+
end
|
9
|
+
subject { driver.instance }
|
10
|
+
|
11
|
+
describe '#configure' do
|
12
|
+
let(:config) do
|
13
|
+
%(
|
14
|
+
aws_key_id AWS_KEY_ID
|
15
|
+
aws_sec_key AWS_SEC_KEY
|
16
|
+
tag TAG
|
17
|
+
region REGION
|
18
|
+
sqs_url http://SQS_URL
|
19
|
+
receive_interval 1
|
20
|
+
max_number_of_messages 10
|
21
|
+
wait_time_seconds 10
|
22
|
+
visibility_timeout 1
|
23
|
+
delete_message true
|
24
|
+
stub_responses true
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'fluentd input configuration settings' do
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.tag).to eq('TAG') }
|
32
|
+
it { expect(subject.region).to eq('REGION') }
|
33
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
34
|
+
it { expect(subject.receive_interval).to eq(1) }
|
35
|
+
it { expect(subject.max_number_of_messages).to eq(10) }
|
36
|
+
it { expect(subject.wait_time_seconds).to eq(10) }
|
37
|
+
it { expect(subject.visibility_timeout).to eq(1) }
|
38
|
+
it { expect(subject.delete_message).to eq(true) }
|
39
|
+
it { expect(subject.stub_responses).to eq(true) }
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'AWS configuration settings' do
|
43
|
+
subject { Aws.config }
|
44
|
+
|
45
|
+
before { driver.instance }
|
46
|
+
|
47
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
48
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
49
|
+
it { expect(subject[:region]).to eq('REGION') }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe '#run' do
|
54
|
+
let(:message_attributes) do
|
55
|
+
{
|
56
|
+
body: 'body',
|
57
|
+
receipt_handle: 'receipt_handle',
|
58
|
+
message_id: 'message_id',
|
59
|
+
md5_of_body: 'md5_of_body',
|
60
|
+
queue_url: 'queue_url',
|
61
|
+
attributes: { 'SenderId' => 'sender_id' }
|
62
|
+
}
|
63
|
+
end
|
64
|
+
let(:queue) { double(:queue, receive_messages: true) }
|
65
|
+
let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
|
66
|
+
let(:messages) { [message] }
|
67
|
+
|
68
|
+
context 'with no delete messages param' do
|
69
|
+
let(:config) do
|
70
|
+
%(
|
71
|
+
tag TAG
|
72
|
+
max_number_of_messages 10
|
73
|
+
wait_time_seconds 10
|
74
|
+
visibility_timeout 1
|
75
|
+
delete_message false
|
76
|
+
)
|
77
|
+
end
|
78
|
+
|
79
|
+
before do
|
80
|
+
allow(subject).to receive(:queue) { queue }
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'parse through messages and emit it' do
|
84
|
+
expect(queue).to receive(:receive_messages)
|
85
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
86
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
87
|
+
expect(message).not_to receive(:delete)
|
88
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
|
89
|
+
|
90
|
+
subject.run
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
context 'with no delete messages param' do
|
95
|
+
let(:config) do
|
96
|
+
%(
|
97
|
+
tag TAG
|
98
|
+
max_number_of_messages 10
|
99
|
+
wait_time_seconds 10
|
100
|
+
visibility_timeout 1
|
101
|
+
delete_message true
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
105
|
+
before do
|
106
|
+
allow(subject).to receive(:queue) { queue }
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'parse through messages and emit it' do
|
110
|
+
expect(queue).to receive(:receive_messages)
|
111
|
+
.with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
|
112
|
+
expect(subject).to receive(:parse_message).with(message) { message_attributes }
|
113
|
+
expect(message).to receive(:delete)
|
114
|
+
expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
|
115
|
+
|
116
|
+
subject.run
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/out_sqs'
|
3
|
+
|
4
|
+
describe Fluent::Plugin::SQSOutput do
|
5
|
+
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::SQSOutput) }
|
6
|
+
subject { driver.instance }
|
7
|
+
|
8
|
+
before { Fluent::Test.setup }
|
9
|
+
|
10
|
+
describe '#configure' do
|
11
|
+
let(:config) do
|
12
|
+
%(
|
13
|
+
aws_key_id AWS_KEY_ID
|
14
|
+
aws_sec_key AWS_SEC_KEY
|
15
|
+
queue_name QUEUE_NAME
|
16
|
+
sqs_url http://SQS_URL
|
17
|
+
create_queue false
|
18
|
+
region REGION
|
19
|
+
delay_seconds 1
|
20
|
+
include_tag false
|
21
|
+
tag_property_name TAG_PROPERTY_NAME
|
22
|
+
message_group_id MESSAGE_GROUP_ID
|
23
|
+
)
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'fluentd output configuration settings' do
|
27
|
+
before { driver.configure(config) }
|
28
|
+
|
29
|
+
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
|
+
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
|
+
it { expect(subject.queue_name).to eq('QUEUE_NAME') }
|
32
|
+
it { expect(subject.sqs_url).to eq('http://SQS_URL') }
|
33
|
+
it { expect(subject.create_queue).to eq(false) }
|
34
|
+
it { expect(subject.region).to eq('REGION') }
|
35
|
+
it { expect(subject.delay_seconds).to eq(1) }
|
36
|
+
it { expect(subject.include_tag).to eq(false) }
|
37
|
+
it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
|
38
|
+
it { expect(subject.message_group_id).to eq('MESSAGE_GROUP_ID') }
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'AWS configuration settings' do
|
42
|
+
subject { Aws.config }
|
43
|
+
|
44
|
+
before do
|
45
|
+
driver.instance
|
46
|
+
driver.configure(config)
|
47
|
+
end
|
48
|
+
|
49
|
+
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
50
|
+
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
51
|
+
it { expect(subject[:region]).to eq('REGION') }
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'using Standard queue' do
|
55
|
+
let(:config) { %( queue_name QUEUE_NAME ) }
|
56
|
+
|
57
|
+
it 'does not raises error' do
|
58
|
+
expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'using FIFO queue and sets message_group_id' do
|
63
|
+
let(:config) do
|
64
|
+
%(
|
65
|
+
queue_name QUEUE_NAME.fifo
|
66
|
+
message_group_id MESSAGE_GROUP_ID
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'does not raise error' do
|
71
|
+
expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
context 'using FIFO queue and does not set message_group_id' do
|
76
|
+
let(:config) { %( queue_name QUEUE_NAME.fifo ) }
|
77
|
+
|
78
|
+
it 'raises error' do
|
79
|
+
expect { driver.configure(config) }.to raise_error(Fluent::ConfigError)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe '#queue' do
|
85
|
+
before { driver.configure(config) }
|
86
|
+
|
87
|
+
context 'when create_queue and queue_name are set' do
|
88
|
+
let(:config) do
|
89
|
+
%(
|
90
|
+
queue_name QUEUE_NAME
|
91
|
+
create_queue true
|
92
|
+
)
|
93
|
+
end
|
94
|
+
let(:resource_instance) { double(:resource_instance) }
|
95
|
+
let(:queue) { double(:queue) }
|
96
|
+
|
97
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
98
|
+
|
99
|
+
it 'calls on create_queue with queue_name' do
|
100
|
+
expect(resource_instance).to receive(:create_queue).with(queue_name: 'QUEUE_NAME') { queue }
|
101
|
+
|
102
|
+
expect(subject.queue).to eq(queue)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
context 'when create_queue is not set but a sqs_url is' do
|
107
|
+
let(:config) do
|
108
|
+
%(
|
109
|
+
queue_name QUEUE_NAME
|
110
|
+
create_queue false
|
111
|
+
sqs_url SQS_URL
|
112
|
+
)
|
113
|
+
end
|
114
|
+
let(:resource_instance) { double(:resource_instance) }
|
115
|
+
let(:queue) { double(:queue) }
|
116
|
+
|
117
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
118
|
+
|
119
|
+
it 'gets queue from sqs_url' do
|
120
|
+
expect(resource_instance).to receive(:queue).with('SQS_URL') { queue }
|
121
|
+
|
122
|
+
expect(subject.queue).to eq(queue)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context 'when create_queue is not set nor sqs_url' do
|
127
|
+
let(:config) do
|
128
|
+
%(
|
129
|
+
queue_name QUEUE_NAME
|
130
|
+
create_queue false
|
131
|
+
)
|
132
|
+
end
|
133
|
+
let(:resource_instance) { double(:resource_instance) }
|
134
|
+
let(:queue) { double(:queue) }
|
135
|
+
|
136
|
+
before { allow(subject).to receive(:resource) { resource_instance } }
|
137
|
+
|
138
|
+
it 'gets queue from queue_name' do
|
139
|
+
expect(resource_instance).to receive(:get_queue_by_name).with(queue_name: 'QUEUE_NAME') { queue }
|
140
|
+
|
141
|
+
expect(subject.queue).to eq(queue)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
describe '#write' do
|
147
|
+
before { driver.configure(config) }
|
148
|
+
|
149
|
+
let(:config) do
|
150
|
+
%(
|
151
|
+
queue_name QUEUE_NAME
|
152
|
+
)
|
153
|
+
end
|
154
|
+
|
155
|
+
let(:record) { {} }
|
156
|
+
let(:body) { double(:body, bytesize: 1) }
|
157
|
+
|
158
|
+
it 'send_messages to queue' do
|
159
|
+
allow(Yajl).to receive(:dump).with(record) { body }
|
160
|
+
|
161
|
+
expect(driver.instance).to receive(:queue).twice.and_return("QUEUE_NAME")
|
162
|
+
expect(subject.queue).to receive(:send_messages).with(entries: [{ id: kind_of(String), message_body: body, delay_seconds: 0 }])
|
163
|
+
|
164
|
+
driver.run(default_tag: "test") do
|
165
|
+
driver.feed(record)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
4
|
+
# loaded once.
|
5
|
+
#
|
6
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
|
+
RSpec.configure do |config|
|
8
|
+
config.run_all_when_everything_filtered = true
|
9
|
+
config.filter_run :focus
|
10
|
+
|
11
|
+
# Run specs in random order to surface order dependencies. If you find an
|
12
|
+
# order dependency and want to debug it, you can fix the order by providing
|
13
|
+
# the seed, which is printed after each run.
|
14
|
+
# --seed 1234
|
15
|
+
config.order = 'random'
|
16
|
+
|
17
|
+
require 'fluent/load'
|
18
|
+
require 'fluent/test'
|
19
|
+
require 'fluent/test/helpers'
|
20
|
+
require 'fluent/test/driver/output'
|
21
|
+
require 'fluent/test/driver/input'
|
22
|
+
|
23
|
+
require 'ostruct'
|
24
|
+
require 'pry'
|
25
|
+
require 'rr'
|
26
|
+
# prevent Test::Unit's AutoRunner from executing during RSpec's rake task
|
27
|
+
Test::Unit.run = true if defined?(Test::Unit) && Test::Unit.respond_to?(:run=)
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,201 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-sqs-tarjei
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 3.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yuri Odagiri
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-04-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.12.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.12.0
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: aws-sdk-sqs
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '1'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '1'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: yajl-ruby
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '1.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: bundler
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: rake
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: rspec
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: rr
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
name: pry
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
type: :development
|
125
|
+
prerelease: false
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: jeweler
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
type: :development
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ">="
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
name: test-unit
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
148
|
+
requirements:
|
149
|
+
- - ">="
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: 3.0.0
|
152
|
+
type: :development
|
153
|
+
prerelease: false
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: 3.0.0
|
159
|
+
description:
|
160
|
+
email: ixixizko@gmail.com
|
161
|
+
executables: []
|
162
|
+
extensions: []
|
163
|
+
extra_rdoc_files:
|
164
|
+
- README.rdoc
|
165
|
+
files:
|
166
|
+
- AUTHORS
|
167
|
+
- README.rdoc
|
168
|
+
- Rakefile
|
169
|
+
- VERSION
|
170
|
+
- lib/fluent/plugin/in_sqs.rb
|
171
|
+
- lib/fluent/plugin/out_sqs.rb
|
172
|
+
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
173
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
174
|
+
- spec/spec_helper.rb
|
175
|
+
homepage: https://github.com/ixixi/fluent-plugin-sqs
|
176
|
+
licenses:
|
177
|
+
- Apache-2.0
|
178
|
+
metadata: {}
|
179
|
+
post_install_message:
|
180
|
+
rdoc_options: []
|
181
|
+
require_paths:
|
182
|
+
- lib
|
183
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
189
|
+
requirements:
|
190
|
+
- - ">="
|
191
|
+
- !ruby/object:Gem::Version
|
192
|
+
version: '0'
|
193
|
+
requirements: []
|
194
|
+
rubygems_version: 3.0.3
|
195
|
+
signing_key:
|
196
|
+
specification_version: 3
|
197
|
+
summary: Amazon SQS input/output plugin for Fluent event collector
|
198
|
+
test_files:
|
199
|
+
- spec/lib/fluent/plugin/in_sqs_spec.rb
|
200
|
+
- spec/lib/fluent/plugin/out_sqs_spec.rb
|
201
|
+
- spec/spec_helper.rb
|