fluent-plugin-sqs 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.rdoc +5 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_sqs.rb +1 -1
- data/lib/fluent/plugin/out_sqs.rb +10 -7
- data/spec/lib/fluent/plugin/out_sqs_spec.rb +42 -5
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9d3cb56ea2f1a833fa07ff05c7d3783a6bebdc0f
|
4
|
+
data.tar.gz: c680daae717c680cbd2408af7280b5e7406ef19f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c53d5a1d95b9629446f5cc4dc8f065d7af2a1d14774e42187736b8057f4dfc56512c28c57dc3b931e484a94f7d888232bbab0923820876491e82f6054a8e5390
|
7
|
+
data.tar.gz: 5c653c3ef79d9ea96c08d0b63e6b6a1e91d8b38ea2f8c487315eafed7df72ba6008bf82c0df10898cd061b5edc91f3a5116533d9c1762baaeb9fdbfacc5ed740
|
data/README.rdoc
CHANGED
@@ -33,10 +33,14 @@ Read events from from amazon SQS.
|
|
33
33
|
aws_sec_key {your_aws_secret_key}
|
34
34
|
|
35
35
|
# following attibutes are optional
|
36
|
-
|
36
|
+
|
37
37
|
create_queue {boolean}
|
38
38
|
region {your_region}
|
39
39
|
|
40
|
+
# following attributes are required if you use FIFO queue
|
41
|
+
|
42
|
+
message_group_id {string}
|
43
|
+
|
40
44
|
### region list ###
|
41
45
|
# Asia Pacific (Tokyo) [Default] : ap-northeast-1
|
42
46
|
# Asia Pacific (Singapore) : ap-southeast-1
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
3.0.0
|
data/lib/fluent/plugin/in_sqs.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'fluent/plugin/output'
|
2
|
-
require 'aws-sdk'
|
2
|
+
require 'aws-sdk-sqs'
|
3
3
|
|
4
4
|
module Fluent::Plugin
|
5
5
|
SQS_BATCH_SEND_MAX_MSGS = 10
|
@@ -24,6 +24,7 @@ module Fluent::Plugin
|
|
24
24
|
config_param :delay_seconds, :integer, default: 0
|
25
25
|
config_param :include_tag, :bool, default: true
|
26
26
|
config_param :tag_property_name, :string, default: '__tag'
|
27
|
+
config_param :message_group_id, :string, default: nil
|
27
28
|
|
28
29
|
config_section :buffer do
|
29
30
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -33,6 +34,10 @@ module Fluent::Plugin
|
|
33
34
|
compat_parameters_convert(conf, :buffer, :inject)
|
34
35
|
super
|
35
36
|
|
37
|
+
if (!@queue_name.nil? && @queue_name.end_with?('.fifo')) || (!@sqs_url.nil? && @sqs_url.end_with?('.fifo'))
|
38
|
+
raise Fluent::ConfigError, 'message_group_id parameter is required for FIFO queue' if @message_group_id.nil?
|
39
|
+
end
|
40
|
+
|
36
41
|
Aws.config = {
|
37
42
|
access_key_id: @aws_key_id,
|
38
43
|
secret_access_key: @aws_sec_key,
|
@@ -100,7 +105,10 @@ module Fluent::Plugin
|
|
100
105
|
"#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
|
101
106
|
"(Truncated message: #{body[0..200]})"
|
102
107
|
else
|
103
|
-
|
108
|
+
id = "#{@tag_property_name}#{SecureRandom.hex(16)}"
|
109
|
+
batch_record = { id: id, message_body: body, delay_seconds: @delay_seconds }
|
110
|
+
batch_record[:message_group_id] = @message_group_id unless @message_group_id.nil?
|
111
|
+
batch_records << batch_record
|
104
112
|
end
|
105
113
|
end
|
106
114
|
|
@@ -111,10 +119,5 @@ module Fluent::Plugin
|
|
111
119
|
end
|
112
120
|
end
|
113
121
|
end
|
114
|
-
|
115
|
-
def generate_id
|
116
|
-
unique_val = ((('a'..'z').to_a + (0..9).to_a)*3).shuffle[0,(rand(10).to_i)].join
|
117
|
-
@tag_property_name + Time.now.to_i.to_s + unique_val
|
118
|
-
end
|
119
122
|
end
|
120
123
|
end
|
@@ -5,10 +5,7 @@ describe Fluent::Plugin::SQSOutput do
|
|
5
5
|
let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::SQSOutput) }
|
6
6
|
subject { driver.instance }
|
7
7
|
|
8
|
-
before
|
9
|
-
Fluent::Test.setup
|
10
|
-
driver.configure(config)
|
11
|
-
end
|
8
|
+
before { Fluent::Test.setup }
|
12
9
|
|
13
10
|
describe '#configure' do
|
14
11
|
let(:config) do
|
@@ -22,10 +19,13 @@ describe Fluent::Plugin::SQSOutput do
|
|
22
19
|
delay_seconds 1
|
23
20
|
include_tag false
|
24
21
|
tag_property_name TAG_PROPERTY_NAME
|
22
|
+
message_group_id MESSAGE_GROUP_ID
|
25
23
|
)
|
26
24
|
end
|
27
25
|
|
28
26
|
context 'fluentd output configuration settings' do
|
27
|
+
before { driver.configure(config) }
|
28
|
+
|
29
29
|
it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
|
30
30
|
it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
|
31
31
|
it { expect(subject.queue_name).to eq('QUEUE_NAME') }
|
@@ -35,20 +35,55 @@ describe Fluent::Plugin::SQSOutput do
|
|
35
35
|
it { expect(subject.delay_seconds).to eq(1) }
|
36
36
|
it { expect(subject.include_tag).to eq(false) }
|
37
37
|
it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
|
38
|
+
it { expect(subject.message_group_id).to eq('MESSAGE_GROUP_ID') }
|
38
39
|
end
|
39
40
|
|
40
41
|
context 'AWS configuration settings' do
|
41
42
|
subject { Aws.config }
|
42
43
|
|
43
|
-
before
|
44
|
+
before do
|
45
|
+
driver.instance
|
46
|
+
driver.configure(config)
|
47
|
+
end
|
44
48
|
|
45
49
|
it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
|
46
50
|
it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
|
47
51
|
it { expect(subject[:region]).to eq('REGION') }
|
48
52
|
end
|
53
|
+
|
54
|
+
context 'using Standard queue' do
|
55
|
+
let(:config) { %( queue_name QUEUE_NAME ) }
|
56
|
+
|
57
|
+
it 'does not raises error' do
|
58
|
+
expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'using FIFO queue and sets message_group_id' do
|
63
|
+
let(:config) do
|
64
|
+
%(
|
65
|
+
queue_name QUEUE_NAME.fifo
|
66
|
+
message_group_id MESSAGE_GROUP_ID
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'does not raise error' do
|
71
|
+
expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
context 'using FIFO queue and does not set message_group_id' do
|
76
|
+
let(:config) { %( queue_name QUEUE_NAME.fifo ) }
|
77
|
+
|
78
|
+
it 'raises error' do
|
79
|
+
expect { driver.configure(config) }.to raise_error(Fluent::ConfigError)
|
80
|
+
end
|
81
|
+
end
|
49
82
|
end
|
50
83
|
|
51
84
|
describe '#queue' do
|
85
|
+
before { driver.configure(config) }
|
86
|
+
|
52
87
|
context 'when create_queue and queue_name are set' do
|
53
88
|
let(:config) do
|
54
89
|
%(
|
@@ -109,6 +144,8 @@ describe Fluent::Plugin::SQSOutput do
|
|
109
144
|
end
|
110
145
|
|
111
146
|
describe '#write' do
|
147
|
+
before { driver.configure(config) }
|
148
|
+
|
112
149
|
let(:config) do
|
113
150
|
%(
|
114
151
|
queue_name QUEUE_NAME
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sqs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuri Odagiri
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-04-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -31,19 +31,19 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: aws-sdk
|
34
|
+
name: aws-sdk-sqs
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '1'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '1'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: yajl-ruby
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -192,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
192
192
|
version: '0'
|
193
193
|
requirements: []
|
194
194
|
rubyforge_project:
|
195
|
-
rubygems_version: 2.
|
195
|
+
rubygems_version: 2.6.11
|
196
196
|
signing_key:
|
197
197
|
specification_version: 3
|
198
198
|
summary: Amazon SQS input/output plugin for Fluent event collector
|