fluent-plugin-sqs 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ac806e59bc8e71ad1eecb7cc1f3a6fc0c7107d8c
4
- data.tar.gz: 01ee1714e4709cadbe7965453abac2d74f664384
3
+ metadata.gz: 9d3cb56ea2f1a833fa07ff05c7d3783a6bebdc0f
4
+ data.tar.gz: c680daae717c680cbd2408af7280b5e7406ef19f
5
5
  SHA512:
6
- metadata.gz: 591e14bd3934e022a5eca41cbf9dd588de7d1976625c2d4eb9a8ae8d32da32eddfa52548a2968d63354473169989d7cc3eb9613713cd960db623e743ed7dd2a8
7
- data.tar.gz: 82b30c47720e3daabe83efcba3dbb91ac24f53dc87439926b0afb36c93387d7646f94806267a2a84dfd86b57aab38119c77d9ac0adcdd76955388570dfc265df
6
+ metadata.gz: c53d5a1d95b9629446f5cc4dc8f065d7af2a1d14774e42187736b8057f4dfc56512c28c57dc3b931e484a94f7d888232bbab0923820876491e82f6054a8e5390
7
+ data.tar.gz: 5c653c3ef79d9ea96c08d0b63e6b6a1e91d8b38ea2f8c487315eafed7df72ba6008bf82c0df10898cd061b5edc91f3a5116533d9c1762baaeb9fdbfacc5ed740
@@ -33,10 +33,14 @@ Read events from from amazon SQS.
33
33
  aws_sec_key {your_aws_secret_key}
34
34
 
35
35
  # following attibutes are optional
36
-  
36
+
37
37
  create_queue {boolean}
38
38
  region {your_region}
39
39
 
40
+ # following attributes are required if you use FIFO queue
41
+
42
+ message_group_id {string}
43
+
40
44
  ### region list ###
41
45
  # Asia Pacific (Tokyo) [Default] : ap-northeast-1
42
46
  # Asia Pacific (Singapore) : ap-southeast-1
data/VERSION CHANGED
@@ -1 +1 @@
1
- 2.1.0
1
+ 3.0.0
@@ -1,5 +1,5 @@
1
1
  require 'fluent/plugin/input'
2
- require 'aws-sdk'
2
+ require 'aws-sdk-sqs'
3
3
 
4
4
  module Fluent::Plugin
5
5
  class SQSInput < Input
@@ -1,5 +1,5 @@
1
1
  require 'fluent/plugin/output'
2
- require 'aws-sdk'
2
+ require 'aws-sdk-sqs'
3
3
 
4
4
  module Fluent::Plugin
5
5
  SQS_BATCH_SEND_MAX_MSGS = 10
@@ -24,6 +24,7 @@ module Fluent::Plugin
24
24
  config_param :delay_seconds, :integer, default: 0
25
25
  config_param :include_tag, :bool, default: true
26
26
  config_param :tag_property_name, :string, default: '__tag'
27
+ config_param :message_group_id, :string, default: nil
27
28
 
28
29
  config_section :buffer do
29
30
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -33,6 +34,10 @@ module Fluent::Plugin
33
34
  compat_parameters_convert(conf, :buffer, :inject)
34
35
  super
35
36
 
37
+ if (!@queue_name.nil? && @queue_name.end_with?('.fifo')) || (!@sqs_url.nil? && @sqs_url.end_with?('.fifo'))
38
+ raise Fluent::ConfigError, 'message_group_id parameter is required for FIFO queue' if @message_group_id.nil?
39
+ end
40
+
36
41
  Aws.config = {
37
42
  access_key_id: @aws_key_id,
38
43
  secret_access_key: @aws_sec_key,
@@ -100,7 +105,10 @@ module Fluent::Plugin
100
105
  "#{SQS_BATCH_SEND_MAX_SIZE} bytes. " \
101
106
  "(Truncated message: #{body[0..200]})"
102
107
  else
103
- batch_records << { id: generate_id, message_body: body, delay_seconds: @delay_seconds }
108
+ id = "#{@tag_property_name}#{SecureRandom.hex(16)}"
109
+ batch_record = { id: id, message_body: body, delay_seconds: @delay_seconds }
110
+ batch_record[:message_group_id] = @message_group_id unless @message_group_id.nil?
111
+ batch_records << batch_record
104
112
  end
105
113
  end
106
114
 
@@ -111,10 +119,5 @@ module Fluent::Plugin
111
119
  end
112
120
  end
113
121
  end
114
-
115
- def generate_id
116
- unique_val = ((('a'..'z').to_a + (0..9).to_a)*3).shuffle[0,(rand(10).to_i)].join
117
- @tag_property_name + Time.now.to_i.to_s + unique_val
118
- end
119
122
  end
120
123
  end
@@ -5,10 +5,7 @@ describe Fluent::Plugin::SQSOutput do
5
5
  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::SQSOutput) }
6
6
  subject { driver.instance }
7
7
 
8
- before do
9
- Fluent::Test.setup
10
- driver.configure(config)
11
- end
8
+ before { Fluent::Test.setup }
12
9
 
13
10
  describe '#configure' do
14
11
  let(:config) do
@@ -22,10 +19,13 @@ describe Fluent::Plugin::SQSOutput do
22
19
  delay_seconds 1
23
20
  include_tag false
24
21
  tag_property_name TAG_PROPERTY_NAME
22
+ message_group_id MESSAGE_GROUP_ID
25
23
  )
26
24
  end
27
25
 
28
26
  context 'fluentd output configuration settings' do
27
+ before { driver.configure(config) }
28
+
29
29
  it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
30
30
  it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
31
31
  it { expect(subject.queue_name).to eq('QUEUE_NAME') }
@@ -35,20 +35,55 @@ describe Fluent::Plugin::SQSOutput do
35
35
  it { expect(subject.delay_seconds).to eq(1) }
36
36
  it { expect(subject.include_tag).to eq(false) }
37
37
  it { expect(subject.tag_property_name).to eq('TAG_PROPERTY_NAME') }
38
+ it { expect(subject.message_group_id).to eq('MESSAGE_GROUP_ID') }
38
39
  end
39
40
 
40
41
  context 'AWS configuration settings' do
41
42
  subject { Aws.config }
42
43
 
43
- before { driver.instance }
44
+ before do
45
+ driver.instance
46
+ driver.configure(config)
47
+ end
44
48
 
45
49
  it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
46
50
  it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
47
51
  it { expect(subject[:region]).to eq('REGION') }
48
52
  end
53
+
54
+ context 'using Standard queue' do
55
+ let(:config) { %( queue_name QUEUE_NAME ) }
56
+
57
+ it 'does not raises error' do
58
+ expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
59
+ end
60
+ end
61
+
62
+ context 'using FIFO queue and sets message_group_id' do
63
+ let(:config) do
64
+ %(
65
+ queue_name QUEUE_NAME.fifo
66
+ message_group_id MESSAGE_GROUP_ID
67
+ )
68
+ end
69
+
70
+ it 'does not raise error' do
71
+ expect { driver.configure(config) }.not_to raise_error(Fluent::ConfigError)
72
+ end
73
+ end
74
+
75
+ context 'using FIFO queue and does not set message_group_id' do
76
+ let(:config) { %( queue_name QUEUE_NAME.fifo ) }
77
+
78
+ it 'raises error' do
79
+ expect { driver.configure(config) }.to raise_error(Fluent::ConfigError)
80
+ end
81
+ end
49
82
  end
50
83
 
51
84
  describe '#queue' do
85
+ before { driver.configure(config) }
86
+
52
87
  context 'when create_queue and queue_name are set' do
53
88
  let(:config) do
54
89
  %(
@@ -109,6 +144,8 @@ describe Fluent::Plugin::SQSOutput do
109
144
  end
110
145
 
111
146
  describe '#write' do
147
+ before { driver.configure(config) }
148
+
112
149
  let(:config) do
113
150
  %(
114
151
  queue_name QUEUE_NAME
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.0
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuri Odagiri
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-06-27 00:00:00.000000000 Z
11
+ date: 2018-04-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -31,19 +31,19 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: aws-sdk
34
+ name: aws-sdk-sqs
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: '2'
39
+ version: '1'
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: '2'
46
+ version: '1'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: yajl-ruby
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -192,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
192
192
  version: '0'
193
193
  requirements: []
194
194
  rubyforge_project:
195
- rubygems_version: 2.5.2
195
+ rubygems_version: 2.6.11
196
196
  signing_key:
197
197
  specification_version: 3
198
198
  summary: Amazon SQS input/output plugin for Fluent event collector