fluent-plugin-google-cloud 0.12.10 → 0.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +99 -71
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +17 -21
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -14
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +269 -213
- data/lib/fluent/plugin/statusz.rb +10 -10
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +199 -136
- data/test/plugin/constants.rb +21 -28
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +37 -25
- data/test/plugin/test_out_google_cloud_grpc.rb +45 -30
- data/test/plugin/utils.rb +9 -8
- metadata +53 -39
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c68b17bd8639e6e1c951fe8a6c910433fde73fbdec4d300c68d6bf3418e956a3
|
4
|
+
data.tar.gz: 55ecbf8dc1fa3c947cb122b7a5f61764e75fc0cfa8186fb76f80134122c2065f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f33ba5df092033dd0b038445eac3212ba8d9fc9ffd6f93ec57e54a952850e754a09c4fdf00bb831d0f1ce6a1d73ed5a2395a53b0f52bdfa4aac8b58b3ddd72f4
|
7
|
+
data.tar.gz: 9e39400962a03ab8babd674984601a3080e524188b4fd015189dfe2f17808397a00160a37923a6cb3e5a51019879ecaf5a226f518291168b867cfa1ccfb9427a
|
data/Gemfile.lock
CHANGED
@@ -1,22 +1,28 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.
|
4
|
+
fluent-plugin-google-cloud (0.13.0)
|
5
5
|
fluentd (= 1.13.3)
|
6
|
-
google-api-client (= 0.
|
7
|
-
google-cloud-logging (=
|
8
|
-
google-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
6
|
+
google-api-client (= 0.53.0)
|
7
|
+
google-cloud-logging (= 2.3.2)
|
8
|
+
google-cloud-monitoring-v3 (= 0.10.0)
|
9
|
+
google-protobuf (= 3.22.1)
|
10
|
+
googleapis-common-protos (= 1.4.0)
|
11
|
+
googleauth (= 1.3.0)
|
12
|
+
grpc (= 1.52.0)
|
13
|
+
json (= 2.6.3)
|
13
14
|
opencensus (= 0.5.0)
|
14
15
|
opencensus-stackdriver (= 0.4.1)
|
15
16
|
|
16
17
|
GEM
|
17
18
|
remote: https://rubygems.org/
|
18
19
|
specs:
|
19
|
-
|
20
|
+
activesupport (7.0.5)
|
21
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
22
|
+
i18n (>= 1.6, < 2)
|
23
|
+
minitest (>= 5.1)
|
24
|
+
tzinfo (~> 2.0)
|
25
|
+
addressable (2.8.4)
|
20
26
|
public_suffix (>= 2.0.2, < 6.0)
|
21
27
|
ast (2.4.2)
|
22
28
|
concurrent-ruby (1.2.2)
|
@@ -31,8 +37,12 @@ GEM
|
|
31
37
|
rexml
|
32
38
|
declarative (0.0.20)
|
33
39
|
docile (1.4.0)
|
34
|
-
faraday (
|
35
|
-
|
40
|
+
faraday (2.7.5)
|
41
|
+
faraday-net_http (>= 2.0, < 3.1)
|
42
|
+
ruby2_keywords (>= 0.0.4)
|
43
|
+
faraday-net_http (3.0.2)
|
44
|
+
faraday-retry (2.2.0)
|
45
|
+
faraday (~> 2.0)
|
36
46
|
fluentd (1.13.3)
|
37
47
|
bundler
|
38
48
|
cool.io (>= 1.4.5, < 2.0.0)
|
@@ -45,75 +55,87 @@ GEM
|
|
45
55
|
tzinfo-data (~> 1.0)
|
46
56
|
webrick (>= 1.4.2, < 1.8.0)
|
47
57
|
yajl-ruby (~> 1.0)
|
48
|
-
gapic-common (0.
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
58
|
+
gapic-common (0.19.1)
|
59
|
+
faraday (>= 1.9, < 3.a)
|
60
|
+
faraday-retry (>= 1.0, < 3.a)
|
61
|
+
google-protobuf (~> 3.14)
|
62
|
+
googleapis-common-protos (>= 1.3.12, < 2.a)
|
63
|
+
googleapis-common-protos-types (>= 1.3.1, < 2.a)
|
64
|
+
googleauth (~> 1.0)
|
65
|
+
grpc (~> 1.36)
|
66
|
+
gems (1.2.0)
|
67
|
+
google-api-client (0.53.0)
|
68
|
+
google-apis-core (~> 0.1)
|
69
|
+
google-apis-generator (~> 0.1)
|
70
|
+
google-apis-core (0.11.0)
|
55
71
|
addressable (~> 2.5, >= 2.5.1)
|
56
|
-
googleauth (>= 0.
|
57
|
-
httpclient (>= 2.8.1, < 3.
|
72
|
+
googleauth (>= 0.16.2, < 2.a)
|
73
|
+
httpclient (>= 2.8.1, < 3.a)
|
58
74
|
mini_mime (~> 1.0)
|
59
75
|
representable (~> 3.0)
|
60
|
-
retriable (>= 2.0, < 4.
|
61
|
-
|
76
|
+
retriable (>= 2.0, < 4.a)
|
77
|
+
rexml
|
78
|
+
webrick
|
79
|
+
google-apis-discovery_v1 (0.14.0)
|
80
|
+
google-apis-core (>= 0.11.0, < 2.a)
|
81
|
+
google-apis-generator (0.12.0)
|
82
|
+
activesupport (>= 5.0)
|
83
|
+
gems (~> 1.2)
|
84
|
+
google-apis-core (>= 0.11.0, < 2.a)
|
85
|
+
google-apis-discovery_v1 (~> 0.5)
|
86
|
+
thor (>= 0.20, < 2.a)
|
62
87
|
google-cloud-core (1.6.0)
|
63
88
|
google-cloud-env (~> 1.0)
|
64
89
|
google-cloud-errors (~> 1.0)
|
65
90
|
google-cloud-env (1.6.0)
|
66
91
|
faraday (>= 0.17.3, < 3.0)
|
67
92
|
google-cloud-errors (1.3.1)
|
68
|
-
google-cloud-logging (
|
93
|
+
google-cloud-logging (2.3.2)
|
69
94
|
concurrent-ruby (~> 1.1)
|
70
|
-
google-cloud-core (~> 1.
|
71
|
-
google-
|
72
|
-
googleapis-common-protos-types (>= 1.0.2)
|
95
|
+
google-cloud-core (~> 1.5)
|
96
|
+
google-cloud-logging-v2 (~> 0.0)
|
73
97
|
stackdriver-core (~> 1.3)
|
74
|
-
google-cloud-
|
75
|
-
gapic-common (
|
98
|
+
google-cloud-logging-v2 (0.10.0)
|
99
|
+
gapic-common (>= 0.19.0, < 2.a)
|
76
100
|
google-cloud-errors (~> 1.0)
|
77
|
-
google-cloud-
|
78
|
-
gapic-common (
|
101
|
+
google-cloud-monitoring-v3 (0.10.0)
|
102
|
+
gapic-common (>= 0.16.0, < 2.a)
|
79
103
|
google-cloud-errors (~> 1.0)
|
80
|
-
google-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
google-protobuf (3.20.0-x86_64-linux)
|
88
|
-
googleapis-common-protos (1.3.10)
|
89
|
-
google-protobuf (~> 3.11)
|
90
|
-
googleapis-common-protos-types (>= 1.0.5, < 2.0)
|
104
|
+
google-cloud-trace-v2 (0.6.0)
|
105
|
+
gapic-common (>= 0.19.0, < 2.a)
|
106
|
+
google-cloud-errors (~> 1.0)
|
107
|
+
google-protobuf (3.22.1)
|
108
|
+
googleapis-common-protos (1.4.0)
|
109
|
+
google-protobuf (~> 3.14)
|
110
|
+
googleapis-common-protos-types (~> 1.2)
|
91
111
|
grpc (~> 1.27)
|
92
|
-
googleapis-common-protos-types (1.
|
112
|
+
googleapis-common-protos-types (1.6.0)
|
93
113
|
google-protobuf (~> 3.14)
|
94
|
-
googleauth (
|
95
|
-
faraday (
|
114
|
+
googleauth (1.3.0)
|
115
|
+
faraday (>= 0.17.3, < 3.a)
|
96
116
|
jwt (>= 1.4, < 3.0)
|
97
117
|
memoist (~> 0.16)
|
98
118
|
multi_json (~> 1.11)
|
99
119
|
os (>= 0.9, < 2.0)
|
100
|
-
signet (
|
101
|
-
grpc (1.
|
102
|
-
google-protobuf (~> 3.
|
120
|
+
signet (>= 0.16, < 2.a)
|
121
|
+
grpc (1.52.0)
|
122
|
+
google-protobuf (~> 3.21)
|
103
123
|
googleapis-common-protos-types (~> 1.0)
|
104
124
|
hashdiff (1.0.1)
|
105
125
|
http_parser.rb (0.7.0)
|
106
126
|
httpclient (2.8.3)
|
107
|
-
|
127
|
+
i18n (1.14.1)
|
128
|
+
concurrent-ruby (~> 1.0)
|
129
|
+
json (2.6.3)
|
108
130
|
jwt (2.7.0)
|
109
131
|
memoist (0.16.2)
|
110
132
|
metaclass (0.0.4)
|
111
133
|
mini_mime (1.1.2)
|
134
|
+
minitest (5.18.0)
|
112
135
|
mocha (1.9.0)
|
113
136
|
metaclass (~> 0.0.1)
|
114
|
-
msgpack (1.
|
137
|
+
msgpack (1.7.1)
|
115
138
|
multi_json (1.15.0)
|
116
|
-
multipart-post (2.3.0)
|
117
139
|
opencensus (0.5.0)
|
118
140
|
opencensus-stackdriver (0.4.1)
|
119
141
|
concurrent-ruby (~> 1.0)
|
@@ -122,31 +144,37 @@ GEM
|
|
122
144
|
google-cloud-trace-v2 (~> 0.1)
|
123
145
|
opencensus (~> 0.5)
|
124
146
|
os (1.1.4)
|
125
|
-
|
147
|
+
parallel (1.23.0)
|
148
|
+
parser (3.2.2.1)
|
126
149
|
ast (~> 2.4.1)
|
127
150
|
power_assert (2.0.3)
|
128
|
-
powerpack (0.1.3)
|
129
151
|
prometheus-client (0.9.0)
|
130
152
|
quantile (~> 0.2.1)
|
131
153
|
public_suffix (5.0.1)
|
132
154
|
quantile (0.2.1)
|
133
|
-
rainbow (
|
134
|
-
|
135
|
-
|
155
|
+
rainbow (3.1.1)
|
156
|
+
rake (13.0.6)
|
157
|
+
regexp_parser (2.8.0)
|
136
158
|
representable (3.2.0)
|
137
159
|
declarative (< 0.1.0)
|
138
160
|
trailblazer-option (>= 0.1.1, < 0.2.0)
|
139
161
|
uber (< 0.2.0)
|
140
162
|
retriable (3.1.2)
|
141
163
|
rexml (3.2.5)
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
rainbow (>=
|
164
|
+
rubocop (1.48.1)
|
165
|
+
json (~> 2.3)
|
166
|
+
parallel (~> 1.10)
|
167
|
+
parser (>= 3.2.0.0)
|
168
|
+
rainbow (>= 2.2.2, < 4.0)
|
169
|
+
regexp_parser (>= 1.8, < 3.0)
|
170
|
+
rexml (>= 3.2.5, < 4.0)
|
171
|
+
rubocop-ast (>= 1.26.0, < 2.0)
|
147
172
|
ruby-progressbar (~> 1.7)
|
148
|
-
unicode-display_width (
|
173
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
174
|
+
rubocop-ast (1.29.0)
|
175
|
+
parser (>= 3.2.1.0)
|
149
176
|
ruby-progressbar (1.13.0)
|
177
|
+
ruby2_keywords (0.0.5)
|
150
178
|
serverengine (2.3.2)
|
151
179
|
sigdump (~> 0.2.2)
|
152
180
|
sigdump (0.2.4)
|
@@ -168,35 +196,35 @@ GEM
|
|
168
196
|
tins (~> 1.0)
|
169
197
|
test-unit (3.3.3)
|
170
198
|
power_assert
|
171
|
-
thor (1.2.
|
199
|
+
thor (1.2.2)
|
172
200
|
tins (1.32.1)
|
173
201
|
sync
|
174
202
|
trailblazer-option (0.1.2)
|
175
203
|
tzinfo (2.0.6)
|
176
204
|
concurrent-ruby (~> 1.0)
|
177
|
-
tzinfo-data (1.
|
205
|
+
tzinfo-data (1.2023.3)
|
178
206
|
tzinfo (>= 1.0.0)
|
179
207
|
uber (0.1.0)
|
180
|
-
unicode-display_width (
|
181
|
-
webmock (3.
|
182
|
-
addressable (>= 2.
|
208
|
+
unicode-display_width (2.4.2)
|
209
|
+
webmock (3.17.1)
|
210
|
+
addressable (>= 2.8.0)
|
183
211
|
crack (>= 0.3.2)
|
184
212
|
hashdiff (>= 0.4.0, < 2.0.0)
|
185
213
|
webrick (1.7.0)
|
186
214
|
yajl-ruby (1.4.3)
|
187
215
|
|
188
216
|
PLATFORMS
|
189
|
-
|
217
|
+
ruby
|
190
218
|
|
191
219
|
DEPENDENCIES
|
192
220
|
coveralls (= 0.8.23)
|
193
221
|
fluent-plugin-google-cloud!
|
194
222
|
mocha (= 1.9.0)
|
195
223
|
prometheus-client (< 0.10)
|
196
|
-
rake (=
|
197
|
-
rubocop (=
|
224
|
+
rake (= 13.0.6)
|
225
|
+
rubocop (= 1.48.1)
|
198
226
|
test-unit (= 3.3.3)
|
199
|
-
webmock (= 3.
|
227
|
+
webmock (= 3.17.1)
|
200
228
|
|
201
229
|
BUNDLED WITH
|
202
|
-
2.4
|
230
|
+
2.1.4
|
data/Rakefile
CHANGED
@@ -28,15 +28,16 @@ task :fix_perms do
|
|
28
28
|
end
|
29
29
|
|
30
30
|
files.each do |file|
|
31
|
-
mode = File.stat(file).mode &
|
32
|
-
next unless mode &
|
31
|
+
mode = File.stat(file).mode & 0o777
|
32
|
+
next unless mode & 0o444 != 0o444
|
33
|
+
|
33
34
|
puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
|
34
|
-
"#{(mode |
|
35
|
-
chmod mode |
|
35
|
+
"#{(mode | 0o444).to_s(8)}"
|
36
|
+
chmod mode | 0o444, file
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
39
40
|
desc 'Run unit tests and RuboCop to check for style violations'
|
40
|
-
task all: [
|
41
|
+
task all: %i[rubocop test fix_perms]
|
41
42
|
|
42
43
|
task default: :all
|
@@ -1,50 +1,46 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-google-cloud'
|
3
|
-
gem.description = <<-
|
3
|
+
gem.description = <<-DESCRIPTION
|
4
4
|
Fluentd plugins for the Stackdriver Logging API, which will make logs
|
5
5
|
viewable in the Stackdriver Logs Viewer and can optionally store them
|
6
6
|
in Google Cloud Storage and/or BigQuery.
|
7
7
|
This is an official Google Ruby gem.
|
8
|
-
|
8
|
+
DESCRIPTION
|
9
9
|
gem.summary = 'fluentd plugins for the Stackdriver Logging API'
|
10
10
|
gem.homepage =
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.
|
13
|
+
gem.version = '0.13.0'
|
14
14
|
gem.authors = ['Stackdriver Agents Team']
|
15
15
|
gem.email = ['stackdriver-agents@google.com']
|
16
|
-
gem.required_ruby_version = Gem::Requirement.new('>= 2.
|
16
|
+
gem.required_ruby_version = Gem::Requirement.new('>= 2.6')
|
17
17
|
|
18
18
|
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
19
19
|
gem.test_files = gem.files.grep(/^(test)/)
|
20
20
|
gem.require_paths = ['lib']
|
21
21
|
|
22
|
-
#
|
22
|
+
# NOTE: In order to update the Fluentd version, please update both here and
|
23
23
|
# also the fluentd version in
|
24
24
|
# https://github.com/GoogleCloudPlatform/google-fluentd/blob/master/config/software/fluentd.rb.
|
25
25
|
gem.add_runtime_dependency 'fluentd', '1.13.3'
|
26
|
-
gem.add_runtime_dependency '
|
27
|
-
gem.add_runtime_dependency '
|
28
|
-
gem.add_runtime_dependency '
|
29
|
-
gem.add_runtime_dependency 'google-cloud-logging', '
|
30
|
-
gem.add_runtime_dependency 'google-
|
31
|
-
gem.add_runtime_dependency '
|
32
|
-
gem.add_runtime_dependency '
|
26
|
+
gem.add_runtime_dependency 'google-api-client', '0.53.0'
|
27
|
+
gem.add_runtime_dependency 'googleapis-common-protos', '1.4.0'
|
28
|
+
gem.add_runtime_dependency 'googleauth', '1.3.0'
|
29
|
+
gem.add_runtime_dependency 'google-cloud-logging', '2.3.2'
|
30
|
+
gem.add_runtime_dependency 'google-cloud-monitoring-v3', '0.10.0'
|
31
|
+
gem.add_runtime_dependency 'google-protobuf', '3.22.1'
|
32
|
+
gem.add_runtime_dependency 'grpc', '1.52.0'
|
33
|
+
gem.add_runtime_dependency 'json', '2.6.3'
|
33
34
|
gem.add_runtime_dependency 'opencensus', '0.5.0'
|
34
35
|
gem.add_runtime_dependency 'opencensus-stackdriver', '0.4.1'
|
35
36
|
|
36
37
|
gem.add_development_dependency 'mocha', '1.9.0'
|
37
38
|
# Keep this the same as in
|
38
39
|
# https://github.com/fluent/fluent-plugin-prometheus/blob/master/fluent-plugin-prometheus.gemspec
|
40
|
+
gem.add_development_dependency 'coveralls', '0.8.23'
|
39
41
|
gem.add_development_dependency 'prometheus-client', '< 0.10'
|
40
|
-
|
41
|
-
|
42
|
-
# initialized.
|
43
|
-
# https://github.com/googleapis/google-auth-library-ruby/issues/227
|
44
|
-
# https://github.com/farcaller/rly/issues/2
|
45
|
-
gem.add_development_dependency 'rake', '10.5.0'
|
46
|
-
gem.add_development_dependency 'rubocop', '0.39.0'
|
42
|
+
gem.add_development_dependency 'rake', '13.0.6'
|
43
|
+
gem.add_development_dependency 'rubocop', '1.48.1'
|
47
44
|
gem.add_development_dependency 'test-unit', '3.3.3'
|
48
|
-
gem.add_development_dependency 'webmock', '3.
|
49
|
-
gem.add_development_dependency 'coveralls', '0.8.23'
|
45
|
+
gem.add_development_dependency 'webmock', '3.17.1'
|
50
46
|
end
|
data/lib/fluent/plugin/common.rb
CHANGED
@@ -12,13 +12,15 @@
|
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
|
+
require 'uri'
|
16
|
+
|
15
17
|
module Common
|
16
18
|
# Constants for service names, resource types and etc.
|
17
19
|
module ServiceConstants
|
18
20
|
APPENGINE_CONSTANTS = {
|
19
21
|
service: 'appengine.googleapis.com',
|
20
22
|
resource_type: 'gae_app',
|
21
|
-
metadata_attributes: %w
|
23
|
+
metadata_attributes: %w[gae_backend_name gae_backend_version]
|
22
24
|
}.freeze
|
23
25
|
COMPUTE_CONSTANTS = {
|
24
26
|
service: 'compute.googleapis.com',
|
@@ -27,9 +29,9 @@ module Common
|
|
27
29
|
GKE_CONSTANTS = {
|
28
30
|
service: 'container.googleapis.com',
|
29
31
|
resource_type: 'gke_container',
|
30
|
-
extra_resource_labels: %w
|
31
|
-
extra_common_labels: %w
|
32
|
-
metadata_attributes: %w
|
32
|
+
extra_resource_labels: %w[namespace_id pod_id container_name],
|
33
|
+
extra_common_labels: %w[namespace_name pod_name],
|
34
|
+
metadata_attributes: %w[cluster-name cluster-location],
|
33
35
|
stream_severity_map: {
|
34
36
|
'stdout' => 'INFO',
|
35
37
|
'stderr' => 'ERROR'
|
@@ -47,12 +49,12 @@ module Common
|
|
47
49
|
DATAFLOW_CONSTANTS = {
|
48
50
|
service: 'dataflow.googleapis.com',
|
49
51
|
resource_type: 'dataflow_step',
|
50
|
-
extra_resource_labels: %w
|
52
|
+
extra_resource_labels: %w[region job_name job_id step_id]
|
51
53
|
}.freeze
|
52
54
|
DATAPROC_CONSTANTS = {
|
53
55
|
service: 'cluster.dataproc.googleapis.com',
|
54
56
|
resource_type: 'cloud_dataproc_cluster',
|
55
|
-
metadata_attributes: %w
|
57
|
+
metadata_attributes: %w[dataproc-cluster-uuid dataproc-cluster-name]
|
56
58
|
}.freeze
|
57
59
|
EC2_CONSTANTS = {
|
58
60
|
service: 'ec2.amazonaws.com',
|
@@ -61,7 +63,7 @@ module Common
|
|
61
63
|
ML_CONSTANTS = {
|
62
64
|
service: 'ml.googleapis.com',
|
63
65
|
resource_type: 'ml_job',
|
64
|
-
extra_resource_labels: %w
|
66
|
+
extra_resource_labels: %w[job_id task_name]
|
65
67
|
}.freeze
|
66
68
|
|
67
69
|
# The map between a subservice name and a resource type.
|
@@ -111,7 +113,7 @@ module Common
|
|
111
113
|
end
|
112
114
|
|
113
115
|
begin
|
114
|
-
open(
|
116
|
+
URI.open("http://#{METADATA_SERVICE_ADDR}", proxy: false) do |f|
|
115
117
|
if f.meta['metadata-flavor'] == 'Google'
|
116
118
|
@log.info 'Detected GCE platform'
|
117
119
|
return Platform::GCE
|
@@ -132,10 +134,10 @@ module Common
|
|
132
134
|
def fetch_gce_metadata(platform, metadata_path)
|
133
135
|
raise "Called fetch_gce_metadata with platform=#{platform}" unless
|
134
136
|
platform == Platform::GCE
|
137
|
+
|
135
138
|
# See https://cloud.google.com/compute/docs/metadata
|
136
|
-
open(
|
137
|
-
|
138
|
-
&:read)
|
139
|
+
URI.open("http://#{METADATA_SERVICE_ADDR}/computeMetadata/v1/#{metadata_path}",
|
140
|
+
'Metadata-Flavor' => 'Google', :proxy => false, &:read)
|
139
141
|
end
|
140
142
|
|
141
143
|
# EC2 Metadata server returns everything in one call. Store it after the
|
@@ -143,10 +145,10 @@ module Common
|
|
143
145
|
def ec2_metadata(platform)
|
144
146
|
raise "Called ec2_metadata with platform=#{platform}" unless
|
145
147
|
platform == Platform::EC2
|
148
|
+
|
146
149
|
unless @ec2_metadata
|
147
150
|
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
148
|
-
open(
|
149
|
-
'/latest/dynamic/instance-identity/document', proxy: false) do |f|
|
151
|
+
URI.open("http://#{METADATA_SERVICE_ADDR}/latest/dynamic/instance-identity/document", proxy: false) do |f|
|
150
152
|
contents = f.read
|
151
153
|
@ec2_metadata = JSON.parse(contents)
|
152
154
|
end
|
@@ -164,6 +166,7 @@ module Common
|
|
164
166
|
missing << 'vm_id' unless vm_id
|
165
167
|
end
|
166
168
|
return if missing.empty?
|
169
|
+
|
167
170
|
raise Fluent::ConfigError,
|
168
171
|
"Unable to obtain metadata parameters: #{missing.join(' ')}"
|
169
172
|
end
|
@@ -203,15 +206,16 @@ module Common
|
|
203
206
|
# 2. If not, try to retrieve it locally.
|
204
207
|
def get_location(platform, zone, use_aws_availability_zone)
|
205
208
|
# Response format: "projects/<number>/zones/<zone>"
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
+
if platform == Platform::GCE
|
210
|
+
zone ||= fetch_gce_metadata(platform,
|
211
|
+
'instance/zone').rpartition('/')[2]
|
212
|
+
end
|
209
213
|
aws_location_key = if use_aws_availability_zone
|
210
214
|
'availabilityZone'
|
211
215
|
else
|
212
216
|
'region'
|
213
217
|
end
|
214
|
-
zone ||=
|
218
|
+
zone ||= "aws:#{ec2_metadata(platform)[aws_location_key]}" if
|
215
219
|
platform == Platform::EC2 &&
|
216
220
|
ec2_metadata(platform).key?(aws_location_key)
|
217
221
|
zone
|
@@ -222,7 +226,8 @@ module Common
|
|
222
226
|
# Create a monitored resource from type and labels.
|
223
227
|
def create_monitored_resource(type, labels)
|
224
228
|
Google::Apis::LoggingV2::MonitoredResource.new(
|
225
|
-
type: type, labels: labels.to_h
|
229
|
+
type: type, labels: labels.to_h
|
230
|
+
)
|
226
231
|
end
|
227
232
|
|
228
233
|
# Retrieve monitored resource via the legacy way.
|
@@ -231,25 +236,30 @@ module Common
|
|
231
236
|
# Metadata Agent. Thus it should be equivalent to what Metadata Agent
|
232
237
|
# returns.
|
233
238
|
def determine_agent_level_monitored_resource_via_legacy(
|
234
|
-
|
239
|
+
platform, subservice_name, detect_subservice, vm_id, zone
|
240
|
+
)
|
235
241
|
resource_type = determine_agent_level_monitored_resource_type(
|
236
|
-
platform, subservice_name, detect_subservice
|
242
|
+
platform, subservice_name, detect_subservice
|
243
|
+
)
|
237
244
|
create_monitored_resource(
|
238
245
|
resource_type,
|
239
246
|
determine_agent_level_monitored_resource_labels(
|
240
|
-
platform, resource_type, vm_id, zone
|
247
|
+
platform, resource_type, vm_id, zone
|
248
|
+
)
|
249
|
+
)
|
241
250
|
end
|
242
251
|
|
243
252
|
# Determine agent level monitored resource type.
|
244
253
|
def determine_agent_level_monitored_resource_type(
|
245
|
-
|
254
|
+
platform, subservice_name, detect_subservice
|
255
|
+
)
|
246
256
|
case platform
|
247
257
|
when Platform::OTHER
|
248
258
|
# Unknown platform will be defaulted to GCE instance.
|
249
|
-
|
259
|
+
COMPUTE_CONSTANTS[:resource_type]
|
250
260
|
|
251
261
|
when Platform::EC2
|
252
|
-
|
262
|
+
EC2_CONSTANTS[:resource_type]
|
253
263
|
|
254
264
|
when Platform::GCE
|
255
265
|
# Resource types determined by subservice_name config.
|
@@ -269,14 +279,15 @@ module Common
|
|
269
279
|
end
|
270
280
|
|
271
281
|
# GCE instance.
|
272
|
-
|
282
|
+
COMPUTE_CONSTANTS[:resource_type]
|
273
283
|
end
|
274
284
|
end
|
275
285
|
|
276
286
|
# Determine agent level monitored resource labels based on the resource
|
277
287
|
# type. Each resource type has its own labels that need to be filled in.
|
278
288
|
def determine_agent_level_monitored_resource_labels(
|
279
|
-
|
289
|
+
platform, type, vm_id, zone
|
290
|
+
)
|
280
291
|
case type
|
281
292
|
# GAE app.
|
282
293
|
when APPENGINE_CONSTANTS[:resource_type]
|
@@ -293,6 +304,7 @@ module Common
|
|
293
304
|
when COMPUTE_CONSTANTS[:resource_type]
|
294
305
|
raise "Cannot construct a #{type} resource without vm_id and zone" \
|
295
306
|
unless vm_id && zone
|
307
|
+
|
296
308
|
return {
|
297
309
|
'instance_id' => vm_id,
|
298
310
|
'zone' => zone
|
@@ -302,6 +314,7 @@ module Common
|
|
302
314
|
when GKE_CONSTANTS[:resource_type]
|
303
315
|
raise "Cannot construct a #{type} resource without vm_id and zone" \
|
304
316
|
unless vm_id && zone
|
317
|
+
|
305
318
|
return {
|
306
319
|
'instance_id' => vm_id,
|
307
320
|
'zone' => zone,
|
@@ -327,6 +340,7 @@ module Common
|
|
327
340
|
when EC2_CONSTANTS[:resource_type]
|
328
341
|
raise "Cannot construct a #{type} resource without vm_id and zone" \
|
329
342
|
unless vm_id && zone
|
343
|
+
|
330
344
|
labels = {
|
331
345
|
'instance_id' => vm_id,
|
332
346
|
'region' => zone
|
@@ -352,9 +366,8 @@ module Common
|
|
352
366
|
# Returns the project ID (as a string) on success, or nil on failure.
|
353
367
|
def self.project_id
|
354
368
|
creds = Google::Auth.get_application_default(LOGGING_SCOPE)
|
355
|
-
if creds.respond_to?(:project_id)
|
356
|
-
|
357
|
-
end
|
369
|
+
return creds.project_id if creds.respond_to?(:project_id) && creds.project_id
|
370
|
+
|
358
371
|
if creds.issuer
|
359
372
|
id = extract_project_id(creds.issuer)
|
360
373
|
return id unless id.nil?
|
@@ -51,9 +51,6 @@ module Fluent
|
|
51
51
|
desc 'The field name for insertIds in the log record.'
|
52
52
|
config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
|
53
53
|
|
54
|
-
# Expose attr_readers for testing.
|
55
|
-
attr_reader :insert_id_key
|
56
|
-
|
57
54
|
def start
|
58
55
|
super
|
59
56
|
@log = $log # rubocop:disable Style/GlobalVars
|
@@ -65,21 +62,11 @@ module Fluent
|
|
65
62
|
@log.info "Initialized the insert ID key to #{@insert_id}."
|
66
63
|
end
|
67
64
|
|
68
|
-
def configure(conf)
|
69
|
-
super
|
70
|
-
end
|
71
|
-
|
72
|
-
def shutdown
|
73
|
-
super
|
74
|
-
end
|
75
|
-
|
76
65
|
# rubocop:disable Lint/UnusedMethodArgument
|
77
66
|
def filter(tag, time, record)
|
78
67
|
# Only generate and add an insertId field if the record is a hash and
|
79
68
|
# the insert ID field is not already set (or set to an empty string).
|
80
|
-
if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
|
81
|
-
record[@insert_id_key] = increment_insert_id
|
82
|
-
end
|
69
|
+
record[@insert_id_key] = increment_insert_id if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
|
83
70
|
record
|
84
71
|
end
|
85
72
|
# rubocop:enable Lint/UnusedMethodArgument
|