fluent-plugin-sqs 1.6.0 → 1.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/lib/fluent/plugin/out_sqs.rb +31 -5
  4. metadata +25 -25
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ee7fa69586cccd89e0adcd923126ea5b38cc593a
4
- data.tar.gz: 83f38689a7e79b9b979e095ea8ca5441a83ecb0b
3
+ metadata.gz: d74f724bc816410b2900572f6723bf07dae8fce1
4
+ data.tar.gz: d4a26deded7814151d8f05b9246bc94c4f75a849
5
5
  SHA512:
6
- metadata.gz: 40a3bf121685e4af9cc468a823979dc35586f246e39b05c8a5d045610567648d0eb5ce816db9ed487c5adcfb45e73862490ff6c4781d63eb6fcd64494870188b
7
- data.tar.gz: 49bddb029cae20349f0c02b31d497df311e92c7b222a54cfbcf969f60fe8105778d002e95ebabacfea308f52cc943b0da5f5fa512840f5ab75bf75584818e4f0
6
+ metadata.gz: a2b7bb4e8e029bc75ac2e8badc364d36b619d324ca659365f30d2a5782d218463f07d4bfa0d7bec1060e8310a133bbbdcedac4dc9b80348e4bc8fe5aecb9a454
7
+ data.tar.gz: c0eace6d1ba82afe058bb69ba67aed318c02db1581b969e2a16da32718676dca12f23d265f9476fc7bfcead05a82314ef299da5d1af3c90c0d66734704b3ad8f
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.6.0
1
+ 1.6.1
data/lib/fluent/plugin/out_sqs.rb CHANGED
@@ -2,8 +2,10 @@ module Fluent
2
2
 
3
3
  require 'aws-sdk'
4
4
 
5
- class SQSOutput < BufferedOutput
5
+ SQS_BATCH_SEND_MAX_MSGS = 10
6
+ SQS_BATCH_SEND_MAX_SIZE = 262144
6
7
 
8
+ class SQSOutput < BufferedOutput
7
9
  Fluent::Plugin.register_output('sqs', self)
8
10
 
9
11
  include SetTagKeyMixin
@@ -54,10 +56,34 @@ module Fluent
54
56
  end
55
57
 
56
58
  def write(chunk)
57
- records = []
58
- chunk.msgpack_each {|record| records << { :message_body => Yajl.dump(record), :delay_seconds => @delay_seconds } }
59
- until records.length <= 0 do
60
- @queue.batch_send(records.slice!(0..9))
59
+ batch_records = []
60
+ batch_size = 0
61
+ send_batches = [batch_records]
62
+
63
+ chunk.msgpack_each do |record|
64
+ body = Yajl.dump(record)
65
+ batch_size += body.bytesize
66
+ if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
67
+ batch_records.length >= SQS_BATCH_SEND_MAX_MSGS then
68
+ batch_records = []
69
+ batch_size = body.bytesize
70
+ send_batches << batch_records
71
+ end
72
+
73
+ if batch_size > SQS_BATCH_SEND_MAX_SIZE then
74
+ log.warn "Could not push message to SQS, payload exceeds "\
75
+ "#{SQS_BATCH_SEND_MAX_SIZE} bytes. "\
76
+ "(Truncated message: #{body[0..200]})"
77
+ else
78
+ batch_records << { :message_body => body, :delay_seconds => @delay_seconds }
79
+ end
80
+ end
81
+
82
+ until send_batches.length <= 0 do
83
+ records = send_batches.shift
84
+ until records.length <= 0 do
85
+ @queue.batch_send(records.slice!(0..9))
86
+ end
61
87
  end
62
88
  end
63
89
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sqs
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.0
4
+ version: 1.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuri Odagiri
@@ -14,140 +14,140 @@ dependencies:
14
14
  name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: 0.12.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: 0.12.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: aws-sdk
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: 1.9.5
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: 1.9.5
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: yajl-ruby
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ~>
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
47
  version: '1.0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '1.0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: bundler
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '>='
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: rake
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: rspec
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - '>='
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - '>='
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: rr
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - '>='
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: '0'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - '>='
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: pry
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - '>='
115
+ - - ">="
116
116
  - !ruby/object:Gem::Version
117
117
  version: '0'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - '>='
122
+ - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: jeweler
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
- - - '>='
129
+ - - ">="
130
130
  - !ruby/object:Gem::Version
131
131
  version: '0'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - '>='
136
+ - - ">="
137
137
  - !ruby/object:Gem::Version
138
138
  version: '0'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: test-unit
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
- - - '>='
143
+ - - ">="
144
144
  - !ruby/object:Gem::Version
145
145
  version: 3.0.0
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
- - - '>='
150
+ - - ">="
151
151
  - !ruby/object:Gem::Version
152
152
  version: 3.0.0
153
153
  description:
@@ -158,13 +158,13 @@ extra_rdoc_files:
158
158
  - README.rdoc
159
159
  files:
160
160
  - AUTHORS
161
+ - README.rdoc
161
162
  - Rakefile
162
163
  - VERSION
163
164
  - lib/fluent/plugin/in_sqs.rb
164
165
  - lib/fluent/plugin/out_sqs.rb
165
166
  - spec/lib/fluent/plugin/in_sqs_spec.rb
166
167
  - spec/spec_helper.rb
167
- - README.rdoc
168
168
  homepage: https://github.com/ixixi/fluent-plugin-sqs
169
169
  licenses: []
170
170
  metadata: {}
@@ -174,17 +174,17 @@ require_paths:
174
174
  - lib
175
175
  required_ruby_version: !ruby/object:Gem::Requirement
176
176
  requirements:
177
- - - '>='
177
+ - - ">="
178
178
  - !ruby/object:Gem::Version
179
179
  version: '0'
180
180
  required_rubygems_version: !ruby/object:Gem::Requirement
181
181
  requirements:
182
- - - '>='
182
+ - - ">="
183
183
  - !ruby/object:Gem::Version
184
184
  version: '0'
185
185
  requirements: []
186
186
  rubyforge_project:
187
- rubygems_version: 2.0.14
187
+ rubygems_version: 2.4.1
188
188
  signing_key:
189
189
  specification_version: 3
190
190
  summary: Amazon SQS input/output plugin for Fluent event collector