fluent-plugin-google-cloud-storage-out 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fluent/plugin/out_google_cloud_storage_out.rb +120 -136
- data/lib/fluent/plugin/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 04a0022a7b4694f01182f15dbdba737a4d2bd006
|
4
|
+
data.tar.gz: 99da050af82790baa08dcc5d35e5702498009651
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d604a2370b3903d940be02d21439612c9dd762934f4d408550581fae2c6baafae0a2fa12a71125363dc9570e0ca64a2df1c662fd91e282f8151382352212e9f
|
7
|
+
data.tar.gz: c326ce6e5db73f466a5187e46b05f4a46f3c8bc31d6e1a60eafb29125e2d6c77e0fb22d75b7c78300fabd2c5ead556d24c085006e78d1b62160a8fc13653f8c3
|
@@ -1,163 +1,147 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
|
3
|
-
|
4
|
-
require "fluent/log"
|
5
|
-
|
6
|
-
require 'googleauth'
|
7
|
-
require 'google/apis/storage_v1'
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
3
|
+
module Fluent
|
4
|
+
require "fluent/log"
|
5
|
+
require "fluent/mixin/config_placeholders"
|
6
|
+
require 'googleauth'
|
7
|
+
require 'google/apis/storage_v1'
|
8
|
+
require 'fluent/plugin/version'
|
9
|
+
class GoogleCloudStorageOut < TimeSlicedOutput
|
10
|
+
Plugin.register_output('google_cloud_storage_out', self)
|
11
|
+
|
12
|
+
Storage = Google::Apis::StorageV1
|
13
|
+
ServiceAccountCredentials = Google::Auth::ServiceAccountCredentials
|
14
|
+
|
15
|
+
extend GoogleCloudStorageOut
|
16
|
+
|
17
|
+
SUPPORTED_COMPRESS = {
|
18
|
+
'gz' => :gz,
|
19
|
+
'gzip' => :gzip,
|
20
|
+
}
|
21
|
+
|
22
|
+
#
|
23
|
+
# Config Parameters
|
24
|
+
#
|
25
|
+
config_set_default :buffer_type, 'file'
|
26
|
+
config_set_default :time_slice_format, '%Y%m%d'
|
27
|
+
config_set_default :flush_interval, nil
|
28
|
+
|
29
|
+
include Mixin::ConfigPlaceholders
|
30
|
+
|
31
|
+
desc "The path of Service Account Json key."
|
32
|
+
config_param :service_account_json_key_path, :string
|
33
|
+
|
34
|
+
desc "The bucket ID for destination for store."
|
35
|
+
config_param :bucket_id, :string
|
36
|
+
|
37
|
+
desc "The directory path for store."
|
38
|
+
config_param :path, :string
|
39
|
+
|
40
|
+
desc "The format of the file content. The default is out_file"
|
41
|
+
config_param :format, :string, :default => 'out_file'
|
42
|
+
|
43
|
+
desc "The tag for out"
|
44
|
+
config_param :default_tag, :string, :default => 'tag_missing'
|
45
|
+
|
46
|
+
desc "Compress flushed file."
|
47
|
+
config_param :compress, :default => nil do |val|
|
48
|
+
c = SUPPORTED_COMPRESS[val]
|
49
|
+
unless c
|
50
|
+
raise ConfigError, "Unsupported compression algorithm '#{val}'"
|
51
|
+
end
|
52
|
+
c
|
53
|
+
end
|
29
54
|
|
30
|
-
|
31
|
-
|
55
|
+
def initialize
|
56
|
+
super
|
57
|
+
require 'zlib'
|
58
|
+
require 'net/http'
|
59
|
+
require 'time'
|
60
|
+
require 'mime-types'
|
61
|
+
end
|
32
62
|
|
33
|
-
|
34
|
-
|
63
|
+
# Define `log` method for v0.10.42 or earlier
|
64
|
+
unless method_defined?(:log)
|
65
|
+
define_method("log") { $log }
|
66
|
+
end
|
35
67
|
|
36
|
-
|
37
|
-
|
68
|
+
def configure(conf)
|
69
|
+
if conf['path']
|
70
|
+
if conf['path'].index('%S')
|
71
|
+
conf['time_slice_format'] = '%Y%m%d%H%M%S'
|
72
|
+
elsif conf['path'].index('%M')
|
73
|
+
conf['time_slice_format'] = '%Y%m%d%H%M'
|
74
|
+
elsif conf['path'].index('%H')
|
75
|
+
conf['time_slice_format'] = '%Y%m%d%H'
|
76
|
+
end
|
77
|
+
end
|
38
78
|
|
39
|
-
|
40
|
-
config_param :format, :string, :default => 'out_file'
|
79
|
+
super
|
41
80
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
unless c
|
46
|
-
raise ConfigError, "Unsupported compression algorithm '#{val}'"
|
81
|
+
@formatter = Plugin.new_formatter(@format)
|
82
|
+
@formatter.configure(conf)
|
83
|
+
prepare_client()
|
47
84
|
end
|
48
|
-
c
|
49
|
-
end
|
50
85
|
|
51
|
-
|
86
|
+
def prepare_client
|
52
87
|
|
53
|
-
|
88
|
+
storage = Storage::StorageService.new
|
89
|
+
scopes = [Storage::AUTH_CLOUD_PLATFORM, Storage::AUTH_DEVSTORAGE_FULL_CONTROL]
|
90
|
+
storage.authorization = ServiceAccountCredentials.make_creds(
|
91
|
+
{
|
92
|
+
:json_key_io => File.open(@service_account_json_key),
|
93
|
+
:scope => scopes
|
94
|
+
}
|
95
|
+
)
|
54
96
|
|
55
|
-
|
56
|
-
super
|
57
|
-
require 'zlib'
|
58
|
-
require 'net/http'
|
59
|
-
require 'time'
|
60
|
-
require 'mime-types'
|
61
|
-
end
|
62
|
-
|
63
|
-
# Define `log` method for v0.10.42 or earlier
|
64
|
-
unless method_defined?(:log)
|
65
|
-
define_method("log") { $log }
|
66
|
-
end
|
67
|
-
|
68
|
-
#def call_google_api(params)
|
69
|
-
# # refresh_auth
|
70
|
-
# if @google_api_client.authorization.expired?
|
71
|
-
# @google_api_client.authorization.fetch_access_token!
|
72
|
-
# end
|
73
|
-
# return @google_api_client.execute(params)
|
74
|
-
#end
|
75
|
-
|
76
|
-
def configure(conf)
|
77
|
-
if conf['path']
|
78
|
-
if conf['path'].index('%S')
|
79
|
-
conf['time_slice_format'] = '%Y%m%d%H%M%S'
|
80
|
-
elsif conf['path'].index('%M')
|
81
|
-
conf['time_slice_format'] = '%Y%m%d%H%M'
|
82
|
-
elsif conf['path'].index('%H')
|
83
|
-
conf['time_slice_format'] = '%Y%m%d%H'
|
84
|
-
end
|
97
|
+
@google_api_client = storage
|
85
98
|
end
|
86
99
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
@formatter.configure(conf)
|
91
|
-
prepare_client()
|
92
|
-
end
|
93
|
-
|
94
|
-
def prepare_client
|
100
|
+
def start
|
101
|
+
super
|
102
|
+
end
|
95
103
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
{
|
100
|
-
:json_key_io => File.open(@service_account_json_key),
|
101
|
-
:scope => scopes
|
102
|
-
}
|
103
|
-
)
|
104
|
+
def shutdown
|
105
|
+
super
|
106
|
+
end
|
104
107
|
|
105
|
-
|
106
|
-
|
108
|
+
def format(tag, time, record)
|
109
|
+
@formatter.format(tag, time, record)
|
110
|
+
end
|
107
111
|
|
108
|
-
|
109
|
-
|
110
|
-
|
112
|
+
def path_format(chunk_key)
|
113
|
+
path = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
114
|
+
log.debug "GCS Path: #{path}"
|
115
|
+
path
|
116
|
+
end
|
111
117
|
|
112
|
-
|
113
|
-
|
114
|
-
|
118
|
+
def send(path, data)
|
119
|
+
mimetype = MIME::Types.type_for(path).first
|
120
|
+
|
121
|
+
io = nil
|
122
|
+
if SUPPORTED_COMPRESS.include?(@compress)
|
123
|
+
io = StringIO.new("")
|
124
|
+
writer = Zlib::GzipWriter.new(io)
|
125
|
+
writer.write(data)
|
126
|
+
writer.finish
|
127
|
+
io.rewind
|
128
|
+
else
|
129
|
+
io = StringIO.new(data)
|
130
|
+
end
|
115
131
|
|
116
|
-
|
117
|
-
@formatter.format(tag, time, record)
|
118
|
-
end
|
132
|
+
media = Google::APIClient::UploadIO.new(io, mimetype.content_type, File.basename(path))
|
119
133
|
|
120
|
-
|
121
|
-
path = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
122
|
-
log.debug "GCS Path: #{path}"
|
123
|
-
path
|
124
|
-
end
|
134
|
+
@gogle_api_client.insert_object(@bucket_id, upload_source: io, name: path, content_type:mimetype_content_type)
|
125
135
|
|
126
|
-
def send(path, data)
|
127
|
-
mimetype = MIME::Types.type_for(path).first
|
128
|
-
|
129
|
-
io = nil
|
130
|
-
if SUPPORTED_COMPRESS.include?(@compress)
|
131
|
-
io = StringIO.new("")
|
132
|
-
writer = Zlib::GzipWriter.new(io)
|
133
|
-
writer.write(data)
|
134
|
-
writer.finish
|
135
|
-
io.rewind
|
136
|
-
else
|
137
|
-
io = StringIO.new(data)
|
138
136
|
end
|
139
137
|
|
140
|
-
|
138
|
+
def write(chunk)
|
139
|
+
gcs_path = path_format(chunk.key)
|
141
140
|
|
142
|
-
|
141
|
+
send(gcs_path, chunk.read)
|
143
142
|
|
144
|
-
|
145
|
-
|
146
|
-
# uploadType: "multipart",
|
147
|
-
# project: @project_id,
|
148
|
-
# bucket: @bucket_id,
|
149
|
-
# name: path
|
150
|
-
# },
|
151
|
-
# body_object: { contentType: media.content_type },
|
152
|
-
# media: media)
|
153
|
-
end
|
154
|
-
|
155
|
-
def write(chunk)
|
156
|
-
gcs_path = path_format(chunk.key)
|
157
|
-
|
158
|
-
send(gcs_path, chunk.read)
|
159
|
-
|
160
|
-
gcs_path
|
143
|
+
gcs_path
|
144
|
+
end
|
161
145
|
end
|
162
146
|
end
|
163
147
|
|