fluent-plugin-gcs 0.3.0 → 0.4.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -6
- data/README.md +41 -1
- data/fluent-plugin-gcs.gemspec +2 -2
- data/lib/fluent/plugin/gcs/version.rb +1 -1
- data/lib/fluent/plugin/out_gcs.rb +48 -12
- metadata +16 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d4e0e0d0018992e06145a4982a26182240998f12
|
4
|
+
data.tar.gz: 5c96a75ae786b74be8f8092ed210ebae68be3ad6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a6d11e853d46efff711d2a0e2f2aeab878ad0923cd7230923bdf90da33ae98d11f7789a76c042faa01151473505e56a98d477904d246a876ca593582491e7f4a
|
7
|
+
data.tar.gz: 0fcffefaf44dc28f0d29d97c6f26e2cd6047ebaa8092eea65726c46637bb9073d31f1d0998ad94de502b1bdd8225f89429a89a9c93b6ed35338495474e0ab499
|
data/CHANGELOG.md
CHANGED
@@ -1,16 +1,32 @@
|
|
1
|
-
##
|
1
|
+
## [Unreleased]
|
2
2
|
|
3
|
-
|
3
|
+
## [0.4.0] - xxxx/xx/xx
|
4
4
|
|
5
|
-
|
5
|
+
New features / Enhancements
|
6
|
+
|
7
|
+
- [Support v0.14 (by @cosmo0920)](https://github.com/daichirata/fluent-plugin-gcs/pull/6)
|
8
|
+
|
9
|
+
## [0.3.0] - 2017/02/28
|
10
|
+
|
11
|
+
New features / Enhancements
|
12
|
+
|
13
|
+
- [Add support for setting a File's storage_class on file creation](https://github.com/daichirata/fluent-plugin-gcs/pull/4)
|
6
14
|
- see also https://cloud.google.com/storage/docs/storage-classes
|
7
15
|
|
8
|
-
|
16
|
+
## [0.2.0] - 2017/01/16
|
17
|
+
|
18
|
+
Bug fixes
|
9
19
|
|
10
|
-
- Remove encryption_key_sha256 parameter.
|
20
|
+
- [Remove encryption_key_sha256 parameter.](https://github.com/daichirata/fluent-plugin-gcs/pull/2)
|
11
21
|
- see also. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-storage/CHANGELOG.md#0230--2016-12-8
|
12
22
|
|
13
|
-
|
23
|
+
## [0.1.1] - 2016/11/28
|
24
|
+
|
25
|
+
New features / Enhancements
|
14
26
|
|
15
27
|
- Add support for `%{hostname}` of object_key_format
|
16
28
|
|
29
|
+
[Unreleased]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.3.0...HEAD
|
30
|
+
[0.3.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.2.0...v0.3.0
|
31
|
+
[0.2.0]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.a...v0.2.0
|
32
|
+
[0.1.1]: https://github.com/daichirata/fluent-plugin-gcs/compare/v0.1.0...v0.1.1
|
data/README.md
CHANGED
@@ -3,14 +3,54 @@
|
|
3
3
|
|
4
4
|
Google Cloud Storage output plugin for [Fluentd](https://github.com/fluent/fluentd).
|
5
5
|
|
6
|
+
## Requirements
|
7
|
+
|
8
|
+
| fluent-plugin-gcs | fluentd | ruby |
|
9
|
+
|--------------------|------------|--------|
|
10
|
+
| >= 0.4.0 | >= v0.14.0 | >= 2.1 |
|
11
|
+
| < 0.4.0 | >= v0.12.0 | >= 1.9 |
|
12
|
+
|
13
|
+
NOTE: fluent-plugin-gcs v0.4.0 is now beta.
|
14
|
+
|
6
15
|
## Installation
|
7
16
|
|
8
17
|
``` shell
|
9
|
-
gem install fluent-plugin-gcs
|
18
|
+
$ gem install fluent-plugin-gcs -v "~> 0.3" --no-document # for fluentd v0.12 or later
|
19
|
+
$ gem install fluent-plugin-gcs -v "0.4.0.beta1" --no-document # for fluentd v0.14 or later
|
10
20
|
```
|
11
21
|
|
12
22
|
## Examples
|
13
23
|
|
24
|
+
### For v0.14 style
|
25
|
+
|
26
|
+
```
|
27
|
+
<match pattern>
|
28
|
+
@type gcs
|
29
|
+
|
30
|
+
project YOUR_PROJECT
|
31
|
+
keyfile YOUR_KEYFILE_PATH
|
32
|
+
bucket YOUR_GCS_BUCKET_NAME
|
33
|
+
object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
34
|
+
path logs/${tag}/%Y/%m/%d/
|
35
|
+
|
36
|
+
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
|
37
|
+
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
|
38
|
+
<buffer tag,time>
|
39
|
+
@type file
|
40
|
+
path /var/log/fluent/gcs
|
41
|
+
timekey 1h # 1 hour partition
|
42
|
+
timekey_wait 10m
|
43
|
+
timekey_use_utc true # use utc
|
44
|
+
</buffer>
|
45
|
+
|
46
|
+
<format>
|
47
|
+
@type json
|
48
|
+
</format>
|
49
|
+
</match>
|
50
|
+
```
|
51
|
+
|
52
|
+
### For v0.12 style
|
53
|
+
|
14
54
|
```
|
15
55
|
<match pattern>
|
16
56
|
@type gcs
|
data/fluent-plugin-gcs.gemspec
CHANGED
@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.add_runtime_dependency "fluentd", "
|
24
|
-
spec.add_runtime_dependency "google-cloud-storage", "~>
|
23
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
|
24
|
+
spec.add_runtime_dependency "google-cloud-storage", "~> 1.1.0"
|
25
25
|
|
26
26
|
spec.add_development_dependency "bundler", "~> 1.13"
|
27
27
|
spec.add_development_dependency "rake", "~> 10.0"
|
@@ -4,11 +4,14 @@ require "socket"
|
|
4
4
|
|
5
5
|
require "fluent/plugin/gcs/object_creator"
|
6
6
|
require "fluent/plugin/gcs/version"
|
7
|
+
require "fluent/plugin/output"
|
7
8
|
|
8
|
-
module Fluent
|
9
|
-
class GCSOutput <
|
9
|
+
module Fluent::Plugin
|
10
|
+
class GCSOutput < Output
|
10
11
|
Fluent::Plugin.register_output("gcs", self)
|
11
12
|
|
13
|
+
helpers :compat_parameters, :formatter, :inject
|
14
|
+
|
12
15
|
def initialize
|
13
16
|
super
|
14
17
|
require "google/cloud/storage"
|
@@ -51,9 +54,21 @@ module Fluent
|
|
51
54
|
config_param :value, :string, default: ""
|
52
55
|
end
|
53
56
|
|
57
|
+
DEFAULT_FORMAT_TYPE = "out_file"
|
58
|
+
|
59
|
+
config_section :format do
|
60
|
+
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
61
|
+
end
|
62
|
+
|
63
|
+
config_section :buffer do
|
64
|
+
config_set_default :chunk_keys, ['time']
|
65
|
+
config_set_default :timekey, (60 * 60 * 24)
|
66
|
+
end
|
67
|
+
|
54
68
|
MAX_HEX_RANDOM_LENGTH = 32
|
55
69
|
|
56
70
|
def configure(conf)
|
71
|
+
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
57
72
|
super
|
58
73
|
|
59
74
|
if @hex_random_length > MAX_HEX_RANDOM_LENGTH
|
@@ -69,10 +84,13 @@ module Fluent
|
|
69
84
|
@object_metadata_hash = @object_metadata.map {|m| [m.key, m.value] }.to_h
|
70
85
|
end
|
71
86
|
|
72
|
-
@formatter =
|
73
|
-
@formatter.configure(conf)
|
87
|
+
@formatter = formatter_create
|
74
88
|
|
75
89
|
@object_creator = Fluent::GCS.discovered_object_creator(@store_as, transcoding: @transcoding)
|
90
|
+
# For backward compatibility
|
91
|
+
# TODO: Remove time_slice_format when end of support compat_parameters
|
92
|
+
@configured_time_slice_format = conf['time_slice_format']
|
93
|
+
@time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
|
76
94
|
end
|
77
95
|
|
78
96
|
def start
|
@@ -89,7 +107,12 @@ module Fluent
|
|
89
107
|
end
|
90
108
|
|
91
109
|
def format(tag, time, record)
|
92
|
-
|
110
|
+
r = inject_values_to_record(tag, time, record)
|
111
|
+
@formatter.format(tag, time, r)
|
112
|
+
end
|
113
|
+
|
114
|
+
def multi_workers_ready?
|
115
|
+
true
|
93
116
|
end
|
94
117
|
|
95
118
|
def write(chunk)
|
@@ -126,22 +149,24 @@ module Fluent
|
|
126
149
|
Digest::MD5.hexdigest(chunk.unique_id)[0...@hex_random_length]
|
127
150
|
end
|
128
151
|
|
129
|
-
def format_path(chunk)
|
130
|
-
now = Time.strptime(chunk.key, @time_slice_format)
|
131
|
-
(@localtime ? now : now.utc).strftime(@path)
|
132
|
-
end
|
133
|
-
|
134
152
|
def generate_path(chunk, i = 0, prev = nil)
|
153
|
+
metadata = chunk.metadata
|
154
|
+
time_slice = if metadata.timekey.nil?
|
155
|
+
''.freeze
|
156
|
+
else
|
157
|
+
@time_slice_with_tz.call(metadata.timekey)
|
158
|
+
end
|
135
159
|
tags = {
|
136
160
|
"%{file_extension}" => @object_creator.file_extension,
|
137
161
|
"%{hex_random}" => hex_random(chunk),
|
138
162
|
"%{hostname}" => Socket.gethostname,
|
139
163
|
"%{index}" => i,
|
140
|
-
"%{path}" =>
|
141
|
-
"%{time_slice}" =>
|
164
|
+
"%{path}" => @path,
|
165
|
+
"%{time_slice}" => time_slice,
|
142
166
|
"%{uuid_flush}" => SecureRandom.uuid,
|
143
167
|
}
|
144
168
|
path = @object_key_format.gsub(Regexp.union(tags.keys), tags)
|
169
|
+
path = extract_placeholders(path, metadata)
|
145
170
|
return path unless @gcs_bucket.find_file(path, @encryption_opts)
|
146
171
|
|
147
172
|
if path == prev
|
@@ -153,5 +178,16 @@ module Fluent
|
|
153
178
|
end
|
154
179
|
generate_path(chunk, i + 1, path)
|
155
180
|
end
|
181
|
+
|
182
|
+
# This is stolen from Fluentd
|
183
|
+
def timekey_to_timeformat(timekey)
|
184
|
+
case timekey
|
185
|
+
when nil then ''
|
186
|
+
when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
|
187
|
+
when 60...3600 then '%Y%m%d%H%M'
|
188
|
+
when 3600...86400 then '%Y%m%d%H'
|
189
|
+
else '%Y%m%d'
|
190
|
+
end
|
191
|
+
end
|
156
192
|
end
|
157
193
|
end
|
metadata
CHANGED
@@ -1,43 +1,49 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-gcs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daichi HIRATA
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.14.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.14.0
|
30
|
+
- - "<"
|
25
31
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
32
|
+
version: '2'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: google-cloud-storage
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
39
|
+
version: 1.1.0
|
34
40
|
type: :runtime
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
46
|
+
version: 1.1.0
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: bundler
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -157,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
157
163
|
version: '0'
|
158
164
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
165
|
requirements:
|
160
|
-
- - "
|
166
|
+
- - ">"
|
161
167
|
- !ruby/object:Gem::Version
|
162
|
-
version:
|
168
|
+
version: 1.3.1
|
163
169
|
requirements: []
|
164
170
|
rubyforge_project:
|
165
171
|
rubygems_version: 2.5.1
|