fluent-plugin-s3 1.0.0.rc2 → 1.0.0.rc3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +76 -12
- data/VERSION +1 -1
- data/lib/fluent/log-ext.rb +12 -0
- data/lib/fluent/plugin/in_s3.rb +16 -2
- data/lib/fluent/plugin/out_s3.rb +22 -9
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 50b7e3d9177ef898ea2ee43c2ec789b2cb3293f2
|
4
|
+
data.tar.gz: 26ad4cbb951285caed1570ce28970344fbdf608a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2bee78cb5c5bc10c157d232107bcfedc7225ee23b49987eb7b2eb1161f998731589d40ecaaabd5ca623462f367d83e3efdfe9ff21b6e95b2030ef7a9b0a4e4ce
|
7
|
+
data.tar.gz: 2f877c094484f090dc2d7a418553af0f3303528625d91e5e9c19350e0c4ea7a013f73926f301e5c8632f9e109fd0e36914ac1a973f714f554d17ff140cbed6a5
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Amazon S3
|
1
|
+
# Amazon S3 plugin for [Fluentd](http://github.com/fluent/fluentd)
|
2
2
|
|
3
3
|
[<img src="https://travis-ci.org/fluent/fluent-plugin-s3.svg?branch=master"
|
4
4
|
alt="Build Status" />](https://travis-ci.org/fluent/fluent-plugin-s3) [<img
|
@@ -20,14 +20,28 @@ the former one is stored in "20110102.gz" file, and latter one in
|
|
20
20
|
SQS queue on the region same as S3 bucket.
|
21
21
|
We must setup SQS queue and S3 event notification before use this plugin.
|
22
22
|
|
23
|
+
## Requirements
|
24
|
+
|
25
|
+
| fluent-plugin-s3 | fluentd | ruby |
|
26
|
+
|-------------------|---------|------|
|
27
|
+
| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
|
28
|
+
| < 1.0.0 | >= v0.12.0 | >= 1.9 |
|
29
|
+
|
30
|
+
NOTE: fluent-plugin-s3 v1.0.0 is now RC. We will release stable v1.0.0 soon.
|
31
|
+
|
23
32
|
## Installation
|
24
33
|
|
25
34
|
Simply use RubyGems:
|
26
35
|
|
27
|
-
gem install fluent-plugin-s3
|
36
|
+
$ gem install fluent-plugin-s3 -v "~> 0.8" --no-document # for fluentd v0.12 or later
|
37
|
+
$ gem install fluent-plugin-s3 -v 1.0.0.rc2 --no-document # for fluentd v0.14 or later
|
28
38
|
|
29
39
|
## Output: Configuration
|
30
40
|
|
41
|
+
### v0.14 style
|
42
|
+
|
43
|
+
With fluentd v0.14 and fluent-plugin-s3 v1.0.0, use new buffer configuration to dynamic parameters.
|
44
|
+
|
31
45
|
<match pattern>
|
32
46
|
@type s3
|
33
47
|
|
@@ -35,15 +49,56 @@ Simply use RubyGems:
|
|
35
49
|
aws_sec_key YOUR_AWS_SECRET_KEY
|
36
50
|
s3_bucket YOUR_S3_BUCKET_NAME
|
37
51
|
s3_region ap-northeast-1
|
52
|
+
|
53
|
+
path logs/${tag}/%Y/%m/%d/
|
38
54
|
s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
55
|
+
|
56
|
+
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
|
57
|
+
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
|
58
|
+
<buffer tag,time>
|
59
|
+
@type file
|
60
|
+
path /var/log/fluent/s3
|
61
|
+
timekey 3600 # 1 hour partition
|
62
|
+
timekey_wait 10m
|
63
|
+
timekey_use_utc true # use utc
|
64
|
+
</buffer>
|
65
|
+
<format>
|
66
|
+
@type json
|
67
|
+
</format>
|
68
|
+
</match>
|
69
|
+
|
70
|
+
For `<buffer>`, you can use any record field in `path` / `s3_object_key_format`.
|
71
|
+
|
72
|
+
path logs/${tag}/${foo}
|
73
|
+
<buffer tag,foo>
|
74
|
+
# parameters...
|
75
|
+
</buffer>
|
76
|
+
|
77
|
+
This configuration doesn't work with fluentd v0.12.
|
78
|
+
|
79
|
+
### v0.12 style
|
80
|
+
|
81
|
+
This configuration works with both fluentd v0.12 and v0.14.
|
82
|
+
|
83
|
+
<match pattern>
|
84
|
+
@type s3
|
85
|
+
|
86
|
+
aws_key_id YOUR_AWS_KEY_ID
|
87
|
+
aws_sec_key YOUR_AWS_SECRET_KEY
|
88
|
+
s3_bucket YOUR_S3_BUCKET_NAME
|
89
|
+
s3_region ap-northeast-1
|
90
|
+
|
39
91
|
path logs/
|
92
|
+
s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
40
93
|
buffer_path /var/log/fluent/s3
|
41
|
-
|
42
94
|
time_slice_format %Y%m%d-%H
|
43
95
|
time_slice_wait 10m
|
44
96
|
utc
|
97
|
+
format json
|
45
98
|
</match>
|
46
99
|
|
100
|
+
If you want to embed tag in `path` / `s3_object_key_format`, you need to use `fluent-plugin-forest` plugin.
|
101
|
+
|
47
102
|
**aws_key_id**
|
48
103
|
|
49
104
|
AWS access key id. This parameter is required when your agent is not
|
@@ -104,7 +159,7 @@ uploaded to S3 in the same time slice.
|
|
104
159
|
* %{file_extention} is always "gz" for
|
105
160
|
now.
|
106
161
|
* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed. If you want to use this placeholder, install `uuidtools` gem first.
|
107
|
-
* %{hostname} is replaced with `Socket.gethostname` result.
|
162
|
+
* %{hostname} is replaced with `Socket.gethostname` result.
|
108
163
|
* %{hex_random} a random hex string that is replaced for each buffer chunk, not
|
109
164
|
assured to be unique. This is used to follow a way of peformance tuning, `Add
|
110
165
|
a Hex Hash Prefix to Key Name`, written in [Request Rate and Performance
|
@@ -136,6 +191,13 @@ You get:
|
|
136
191
|
"log/events/ts=20130111-23/events_1.gz"
|
137
192
|
"log/events/ts=20130112-00/events_0.gz"
|
138
193
|
|
194
|
+
NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result by using [configuration's embedded ruby code feature](http://docs.fluentd.org/articles/config-file#embedded-ruby-code).
|
195
|
+
|
196
|
+
s3_object_key_format %{path}%{time_slice}_%{hostname}%{index}.%{file_extension}
|
197
|
+
s3_object_key_format "%{path}%{time_slice}_#{Socket.gethostname}%{index}.%{file_extension}"
|
198
|
+
|
199
|
+
Above two configurations are same. The important point is wrapping `""` is needed for `#{Socket.gethostname}`.
|
200
|
+
|
139
201
|
**force_path_style**
|
140
202
|
|
141
203
|
:force_path_style (Boolean) — default: false — When set to true, the
|
@@ -157,7 +219,7 @@ archive format on S3. You can use serveral format:
|
|
157
219
|
|
158
220
|
See `Use your compression algorithm` section for adding another format.
|
159
221
|
|
160
|
-
|
222
|
+
**`<format>` or format**
|
161
223
|
|
162
224
|
Change one line format in the S3 object. Supported formats are "out_file",
|
163
225
|
"json", "ltsv" and "single_value". See also [official Formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
|
@@ -472,7 +534,7 @@ Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
|
|
472
534
|
## Input: Configuration
|
473
535
|
|
474
536
|
<source>
|
475
|
-
type s3
|
537
|
+
@type s3
|
476
538
|
|
477
539
|
aws_key_id YOUR_AWS_KEY_ID
|
478
540
|
aws_sec_key YOUR_AWS_SECRET_KEY
|
@@ -571,14 +633,15 @@ roles](http://docs.aws.amazon.com/IAM/latest/UserGuide/WorkingWithRoles.html)
|
|
571
633
|
with a properly configured IAM policy are preferred over embedding access keys
|
572
634
|
on EC2 instances.
|
573
635
|
|
574
|
-
## Use your compression algorithm
|
636
|
+
## Use your (de)compression algorithm
|
575
637
|
|
576
638
|
s3 plugin has plugabble compression mechanizm like Fleuntd's input / output
|
577
|
-
plugin. If you set 'store_as xxx',
|
578
|
-
`fluent/plugin/s3_compressor_xxx.rb
|
579
|
-
|
639
|
+
plugin. If you set 'store_as xxx', `out_s3` plugin searches
|
640
|
+
`fluent/plugin/s3_compressor_xxx.rb` and `in_s3` plugin searches
|
641
|
+
`fluent/plugin/s3_extractor_xxx.rb`. You can define your (de)compression with
|
642
|
+
'S3Output::Compressor'/`S3Input::Extractor` classes. Compressor API is here:
|
580
643
|
|
581
|
-
module Fluent
|
644
|
+
module Fluent # Since fluent-plugin-s3 v1.0.0 or later, use Fluent::Plugin instead of Fluent
|
582
645
|
class S3Output
|
583
646
|
class XXXCompressor < Compressor
|
584
647
|
S3Output.register_compressor('xxx', self)
|
@@ -601,7 +664,8 @@ plugin. If you set 'store_as xxx', s3 plugin searches
|
|
601
664
|
end
|
602
665
|
end
|
603
666
|
|
604
|
-
|
667
|
+
`Extractor` is similar to `Compressor`
|
668
|
+
See bundled `Compressor`/`Extractor` classes for more detail.
|
605
669
|
|
606
670
|
## Website, license, et. al.
|
607
671
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.0.
|
1
|
+
1.0.0.rc3
|
data/lib/fluent/plugin/in_s3.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
require 'fluent/input'
|
1
|
+
require 'fluent/plugin/input'
|
2
|
+
require 'fluent/log-ext'
|
2
3
|
|
3
4
|
require 'aws-sdk-resources'
|
4
5
|
require 'zlib'
|
@@ -6,7 +7,7 @@ require 'time'
|
|
6
7
|
require 'tempfile'
|
7
8
|
|
8
9
|
module Fluent::Plugin
|
9
|
-
class S3Input <
|
10
|
+
class S3Input < Input
|
10
11
|
Fluent::Plugin.register_input('s3', self)
|
11
12
|
|
12
13
|
helpers :compat_parameters, :parser, :thread
|
@@ -101,6 +102,7 @@ module Fluent::Plugin
|
|
101
102
|
super
|
102
103
|
|
103
104
|
s3_client = create_s3_client
|
105
|
+
log.debug("Succeeded to create S3 client")
|
104
106
|
@s3 = Aws::S3::Resource.new(client: s3_client)
|
105
107
|
@bucket = @s3.bucket(@s3_bucket)
|
106
108
|
|
@@ -109,8 +111,10 @@ module Fluent::Plugin
|
|
109
111
|
check_apikeys if @check_apikey_on_start
|
110
112
|
|
111
113
|
sqs_client = create_sqs_client
|
114
|
+
log.debug("Succeeded to create SQS client")
|
112
115
|
response = sqs_client.get_queue_url(queue_name: @sqs.queue_name)
|
113
116
|
sqs_queue_url = response.queue_url
|
117
|
+
log.debug("Succeeded to get SQS queue URL")
|
114
118
|
|
115
119
|
@poller = Aws::SQS::QueuePoller.new(sqs_queue_url, client: sqs_client)
|
116
120
|
|
@@ -135,6 +139,7 @@ module Fluent::Plugin
|
|
135
139
|
@poller.poll(options) do |message|
|
136
140
|
begin
|
137
141
|
body = Yajl.load(message.body)
|
142
|
+
log.debug(body)
|
138
143
|
next unless body["Records"] # skip test queue
|
139
144
|
|
140
145
|
process(body)
|
@@ -185,6 +190,10 @@ module Fluent::Plugin
|
|
185
190
|
options = setup_credentials
|
186
191
|
options[:region] = @s3_region if @s3_region
|
187
192
|
options[:proxy_uri] = @proxy_uri if @proxy_uri
|
193
|
+
log.on_trace do
|
194
|
+
options[:http_wire_trace] = true
|
195
|
+
options[:logger] = log
|
196
|
+
end
|
188
197
|
|
189
198
|
Aws::S3::Client.new(options)
|
190
199
|
end
|
@@ -192,12 +201,17 @@ module Fluent::Plugin
|
|
192
201
|
def create_sqs_client
|
193
202
|
options = setup_credentials
|
194
203
|
options[:region] = @s3_region if @s3_region
|
204
|
+
log.on_trace do
|
205
|
+
options[:http_wire_trace] = true
|
206
|
+
options[:logger] = log
|
207
|
+
end
|
195
208
|
|
196
209
|
Aws::SQS::Client.new(options)
|
197
210
|
end
|
198
211
|
|
199
212
|
def check_apikeys
|
200
213
|
@bucket.objects.first
|
214
|
+
log.debug("Succeeded to verify API keys")
|
201
215
|
rescue => e
|
202
216
|
raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
|
203
217
|
end
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
|
-
require 'fluent/output'
|
1
|
+
require 'fluent/plugin/output'
|
2
|
+
require 'fluent/log-ext'
|
2
3
|
require 'aws-sdk-resources'
|
3
4
|
require 'zlib'
|
4
5
|
require 'time'
|
5
6
|
require 'tempfile'
|
6
7
|
|
7
8
|
module Fluent::Plugin
|
8
|
-
class S3Output <
|
9
|
+
class S3Output < Output
|
9
10
|
Fluent::Plugin.register_output('s3', self)
|
10
11
|
|
11
12
|
helpers :compat_parameters, :formatter, :inject
|
@@ -113,6 +114,11 @@ module Fluent::Plugin
|
|
113
114
|
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
114
115
|
end
|
115
116
|
|
117
|
+
config_section :buffer do
|
118
|
+
config_set_default :chunk_keys, ['time']
|
119
|
+
config_set_default :timekey, (60 * 60 * 24)
|
120
|
+
end
|
121
|
+
|
116
122
|
attr_reader :bucket
|
117
123
|
|
118
124
|
MAX_HEX_RANDOM_LENGTH = 16
|
@@ -166,6 +172,10 @@ module Fluent::Plugin
|
|
166
172
|
options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
|
167
173
|
options[:signature_version] = @signature_version unless @signature_version.nil?
|
168
174
|
options[:ssl_verify_peer] = @ssl_verify_peer
|
175
|
+
log.on_trace do
|
176
|
+
options[:http_wire_trace] = true
|
177
|
+
options[:logger] = log
|
178
|
+
end
|
169
179
|
|
170
180
|
s3_client = Aws::S3::Client.new(options)
|
171
181
|
@s3 = Aws::S3::Resource.new(client: s3_client)
|
@@ -188,19 +198,22 @@ module Fluent::Plugin
|
|
188
198
|
|
189
199
|
def write(chunk)
|
190
200
|
i = 0
|
201
|
+
metadata = chunk.metadata
|
191
202
|
previous_path = nil
|
192
203
|
time_slice_format = @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey'])
|
193
|
-
time_slice =
|
204
|
+
time_slice = if metadata.timekey.nil?
|
205
|
+
''.freeze
|
206
|
+
else
|
207
|
+
Time.at(metadata.timekey).utc.strftime(time_slice_format)
|
208
|
+
end
|
194
209
|
|
195
210
|
if @check_object
|
196
211
|
begin
|
197
|
-
path = extract_placeholders(@path, chunk.metadata)
|
198
|
-
|
199
212
|
@values_for_s3_object_chunk[chunk.unique_id] ||= {
|
200
213
|
"%{hex_random}" => hex_random(chunk),
|
201
214
|
}
|
202
215
|
values_for_s3_object_key = {
|
203
|
-
"%{path}" => path,
|
216
|
+
"%{path}" => @path,
|
204
217
|
"%{time_slice}" => time_slice,
|
205
218
|
"%{file_extension}" => @compressor.ext,
|
206
219
|
"%{index}" => i,
|
@@ -208,6 +221,7 @@ module Fluent::Plugin
|
|
208
221
|
values_for_s3_object_key["%{uuid_flush}".freeze] = uuid_random if @uuid_flush_enabled
|
209
222
|
|
210
223
|
s3path = @s3_object_key_format.gsub(%r(%{[^}]+}), values_for_s3_object_key)
|
224
|
+
s3path = extract_placeholders(s3path, metadata)
|
211
225
|
if (i > 0) && (s3path == previous_path)
|
212
226
|
if @overwrite
|
213
227
|
log.warn "#{s3path} already exists, but will overwrite"
|
@@ -227,19 +241,18 @@ module Fluent::Plugin
|
|
227
241
|
hms_slicer = Time.now.utc.strftime("%H%M%S")
|
228
242
|
end
|
229
243
|
|
230
|
-
path = extract_placeholders(@path, chunk.metadata)
|
231
|
-
|
232
244
|
@values_for_s3_object_chunk[chunk.unique_id] ||= {
|
233
245
|
"%{hex_random}" => hex_random(chunk),
|
234
246
|
}
|
235
247
|
values_for_s3_object_key = {
|
236
|
-
"%{path}" => path,
|
248
|
+
"%{path}" => @path,
|
237
249
|
"%{time_slice}" => time_slice,
|
238
250
|
"%{file_extension}" => @compressor.ext,
|
239
251
|
}.merge!(@values_for_s3_object_chunk[chunk.unique_id])
|
240
252
|
values_for_s3_object_key["%{uuid_flush}".freeze] = uuid_random if @uuid_flush_enabled
|
241
253
|
|
242
254
|
s3path = @s3_object_key_format.gsub(%r(%{[^}]+}), values_for_s3_object_key)
|
255
|
+
s3path = extract_placeholders(s3path, metadata)
|
243
256
|
end
|
244
257
|
|
245
258
|
tmp = Tempfile.new("s3-")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.rc3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-03-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -124,6 +124,7 @@ files:
|
|
124
124
|
- VERSION
|
125
125
|
- appveyor.yml
|
126
126
|
- fluent-plugin-s3.gemspec
|
127
|
+
- lib/fluent/log-ext.rb
|
127
128
|
- lib/fluent/plugin/in_s3.rb
|
128
129
|
- lib/fluent/plugin/out_s3.rb
|
129
130
|
- lib/fluent/plugin/s3_compressor_gzip_command.rb
|
@@ -154,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
154
155
|
version: 1.3.1
|
155
156
|
requirements: []
|
156
157
|
rubyforge_project:
|
157
|
-
rubygems_version: 2.
|
158
|
+
rubygems_version: 2.6.8
|
158
159
|
signing_key:
|
159
160
|
specification_version: 4
|
160
161
|
summary: Amazon S3 output plugin for Fluentd event collector
|