fluent-plugin-google-cloud 0.8.4 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,24 @@
1
+ <source>
2
+ @type syslog
3
+ port 514
4
+ protocol_type tcp
5
+ bind 127.0.0.1
6
+ tag syslog
7
+ </source>
8
+
9
+ <source>
10
+ @type tail
11
+ path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
12
+ pos_file /var/lib/google-fluentd/pos/apache-access.pos
13
+ tag apache-access
14
+ </source>
15
+
16
+ <filter **>
17
+ @type add_insert_ids
18
+ </filter>
19
+
20
+ <match **>
21
+ @type google_cloud
22
+ adjust_invalid_timestamps true
23
+ autoformat_stackdriver_trace true
24
+ </match>
@@ -0,0 +1,40 @@
1
+ <source>
2
+ @type syslog
3
+ port 514
4
+ protocol_type tcp
5
+ bind 127.0.0.1
6
+ tag syslog
7
+ </source>
8
+
9
+ <source>
10
+ @type tail
11
+ path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
12
+ pos_file /var/lib/google-fluentd/pos/apache-access.pos
13
+ tag apache-access
14
+ </source>
15
+
16
+ <filter **>
17
+ @type add_insert_ids
18
+ </filter>
19
+
20
+ <match **>
21
+ @type google_cloud
22
+ adjust_invalid_timestamps true
23
+ autoformat_stackdriver_trace false
24
+ coerce_to_utf8 true
25
+ </match>
26
+
27
+ <filter **>
28
+ @type some_custom_filter
29
+ </filter>
30
+
31
+ <filter **>
32
+ @type record_transformer
33
+ <record>
34
+ host_param "#{Socket.gethostname}"
35
+ </record>
36
+ </filter>
37
+
38
+ <match "app.#{ENV['FLUENTD_TAG']}">
39
+ @type stdout
40
+ </match>
@@ -16,19 +16,6 @@ require 'fluent/engine'
16
16
  require 'fluent/event'
17
17
  require 'fluent/test/input_test'
18
18
 
19
- module Fluent
20
- module Test
21
- # rubocop:disable Style/ClassVars
22
- class BufferedOutputTestDriver < InputTestDriver
23
- @@run_method = BufferedOutputTestDriver.instance_method(:run)
24
- def run(num_waits = 0)
25
- @@run_method.bind(self).call(num_waits)
26
- end
27
- end
28
- # rubocop:enable Style/ClassVars
29
- end
30
- end
31
-
32
19
  module Fluent
33
20
  module Test
34
21
  # Similar to the standard BufferedOutputTestDriver, but allows multiple tags
@@ -46,7 +33,7 @@ module Fluent
46
33
  self
47
34
  end
48
35
 
49
- def run(num_waits = 0)
36
+ def run(num_waits = 10)
50
37
  result = nil
51
38
  super(num_waits) do
52
39
  chunk = @instance.buffer.generate_chunk(
@@ -0,0 +1,200 @@
1
+ # Copyright 2020 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require_relative '../helper'
16
+ require_relative 'asserts'
17
+ require_relative 'constants'
18
+ require_relative 'utils'
19
+
20
+ require 'fluent/test/driver/filter'
21
+ require 'fluent/plugin/filter_analyze_config'
22
+
23
+ # Unit tests for filter_analyze_config plugin.
24
+ class FilterAnalyzeConfigTest < Test::Unit::TestCase
25
+ include Asserts
26
+ include Constants
27
+ include Fluent::AnalyzeConfigFilter::Constants
28
+ include Utils
29
+
30
+ APPLICATION_DEFAULT_CONFIG = ''.freeze
31
+
32
+ def setup
33
+ Fluent::Test.setup
34
+ delete_env_vars
35
+ end
36
+
37
+ def test_config_file_does_not_exist
38
+ # By default, the FilterTestDriver.new does not set up a config file at:
39
+ # /etc/google-fluentd/google-fluentd.conf. The plugin should still proceed.
40
+ create_driver
41
+ # No exceptions were thrown.
42
+ end
43
+
44
+ def test_analyze_config
45
+ setup_auth_stubs('https://oauth2.googleapis.com/token')
46
+ setup_gce_metadata_stubs
47
+ [
48
+ [CONFIG_ANALYZE_CONFIG_PROMETHEUS,
49
+ method(:assert_prometheus_metric_value)],
50
+ [CONFIG_ANALYZE_CONFIG_OPENCENSUS,
51
+ method(:assert_opencensus_metric_value)]
52
+ ].each do |config, assert_metric_value|
53
+ clear_metrics
54
+ create_driver(config)
55
+
56
+ # Default plugins, with default config.
57
+ assert_metric_value.call(
58
+ :stackdriver_enabled_plugins,
59
+ 1,
60
+ plugin_name: 'source/syslog/tcp',
61
+ is_default_plugin: true,
62
+ has_default_value: true,
63
+ has_ruby_snippet: false)
64
+ assert_metric_value.call(
65
+ :stackdriver_enabled_plugins,
66
+ 1,
67
+ plugin_name: 'source/tail/apache-access',
68
+ is_default_plugin: true,
69
+ has_default_value: true,
70
+ has_ruby_snippet: false)
71
+ assert_metric_value.call(
72
+ :stackdriver_enabled_plugins,
73
+ 1,
74
+ plugin_name: 'filter/add_insert_ids',
75
+ is_default_plugin: true,
76
+ has_default_value: true,
77
+ has_ruby_snippet: false)
78
+
79
+ # Default plugins, with custom config.
80
+ assert_metric_value.call(
81
+ :stackdriver_enabled_plugins,
82
+ 1,
83
+ plugin_name: 'match/google_cloud',
84
+ is_default_plugin: true,
85
+ has_default_value: false,
86
+ has_ruby_snippet: false)
87
+
88
+ # Custom plugins, some with embedded Ruby.
89
+ assert_metric_value.call(
90
+ :stackdriver_enabled_plugins,
91
+ 1,
92
+ plugin_name: 'filter',
93
+ is_default_plugin: false,
94
+ has_default_value: false,
95
+ has_ruby_snippet: false)
96
+ assert_metric_value.call(
97
+ :stackdriver_enabled_plugins,
98
+ 1,
99
+ plugin_name: 'filter/record_transformer',
100
+ is_default_plugin: false,
101
+ has_default_value: false,
102
+ has_ruby_snippet: true)
103
+ assert_metric_value.call(
104
+ :stackdriver_enabled_plugins,
105
+ 1,
106
+ plugin_name: 'match/stdout',
107
+ is_default_plugin: false,
108
+ has_default_value: false,
109
+ has_ruby_snippet: true)
110
+
111
+ # For out_google_cloud, 3 params are present.
112
+ assert_metric_value.call(
113
+ :stackdriver_config_usage,
114
+ 1,
115
+ plugin_name: 'google_cloud',
116
+ param: 'adjust_invalid_timestamps',
117
+ is_present: true,
118
+ has_default_value: true)
119
+ assert_metric_value.call(
120
+ :stackdriver_config_usage,
121
+ 1,
122
+ plugin_name: 'google_cloud',
123
+ param: 'autoformat_stackdriver_trace',
124
+ is_present: true,
125
+ has_default_value: false)
126
+ assert_metric_value.call(
127
+ :stackdriver_config_usage,
128
+ 1,
129
+ plugin_name: 'google_cloud',
130
+ param: 'coerce_to_utf8',
131
+ is_present: true,
132
+ has_default_value: false)
133
+ # The remaining "google_cloud" params are not present.
134
+ # The are no params for "detect_exceptions".
135
+ %w(
136
+ auth_method
137
+ detect_json
138
+ enable_monitoring
139
+ gcm_service_address
140
+ grpc_compression_algorithm
141
+ http_request_key
142
+ insert_id_key
143
+ label_map
144
+ labels
145
+ labels_key
146
+ logging_api_url
147
+ monitoring_type
148
+ non_utf8_replacement_string
149
+ operation_key
150
+ private_key_email
151
+ private_key_passphrase
152
+ private_key_path
153
+ project_id
154
+ source_location_key
155
+ span_id_key
156
+ statusz_port
157
+ trace_key
158
+ trace_sampled_key
159
+ use_grpc
160
+ use_metadata_service
161
+ vm_id
162
+ vm_name
163
+ zone
164
+ ).each do |p|
165
+ assert_metric_value.call(
166
+ :stackdriver_config_usage,
167
+ 1,
168
+ plugin_name: 'google_cloud',
169
+ param: p,
170
+ is_present: false,
171
+ has_default_value: false)
172
+ end
173
+
174
+ # We also export values for the bools.
175
+ assert_metric_value.call(
176
+ :stackdriver_config_bool_values,
177
+ 1,
178
+ plugin_name: 'google_cloud',
179
+ param: 'adjust_invalid_timestamps',
180
+ value: true)
181
+ assert_metric_value.call(
182
+ :stackdriver_config_bool_values,
183
+ 1,
184
+ plugin_name: 'google_cloud',
185
+ param: 'autoformat_stackdriver_trace',
186
+ value: false)
187
+ assert_metric_value.call(
188
+ :stackdriver_config_bool_values,
189
+ 1,
190
+ plugin_name: 'google_cloud',
191
+ param: 'coerce_to_utf8',
192
+ value: true)
193
+ end
194
+ end
195
+
196
+ def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
197
+ Fluent::Test::FilterTestDriver.new(
198
+ Fluent::AnalyzeConfigFilter).configure(conf, true)
199
+ end
200
+ end
@@ -0,0 +1,147 @@
1
+ # Copyright 2020 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require_relative 'constants'
16
+
17
+ require 'prometheus/client'
18
+ require 'webmock/test_unit'
19
+
20
+ module Utils
21
+ include Constants
22
+
23
+ def delete_env_vars
24
+ # delete environment variables that googleauth uses to find credentials.
25
+ ENV.delete(CREDENTIALS_PATH_ENV_VAR)
26
+ # service account env.
27
+ ENV.delete(PRIVATE_KEY_VAR)
28
+ ENV.delete(CLIENT_EMAIL_VAR)
29
+ ENV.delete(PROJECT_ID_VAR)
30
+ # authorized_user env.
31
+ ENV.delete(CLIENT_ID_VAR)
32
+ ENV.delete(CLIENT_SECRET_VAR)
33
+ ENV.delete(REFRESH_TOKEN_VAR)
34
+ # home var, which is used to find $HOME/.gcloud/...
35
+ ENV.delete('HOME')
36
+ end
37
+
38
+ def stub_metadata_request(metadata_path, response_body)
39
+ stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
40
+ metadata_path)
41
+ .to_return(body: response_body, status: 200,
42
+ headers: { 'Content-Length' => response_body.length })
43
+ end
44
+
45
+ def setup_no_metadata_service_stubs
46
+ # Simulate a machine with no metadata service present
47
+ stub_request(:any, %r{http://169.254.169.254/.*})
48
+ .to_raise(Errno::EHOSTUNREACH)
49
+ end
50
+
51
+ def setup_gce_metadata_stubs
52
+ # Stub the root, used for platform detection by the plugin and 'googleauth'.
53
+ stub_request(:get, 'http://169.254.169.254')
54
+ .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
55
+
56
+ # Create stubs for all the GCE metadata lookups the agent needs to make.
57
+ stub_metadata_request('project/project-id', PROJECT_ID)
58
+ stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
59
+ stub_metadata_request('instance/id', VM_ID)
60
+ stub_metadata_request('instance/attributes/',
61
+ "attribute1\nattribute2\nattribute3")
62
+
63
+ # Used by 'googleauth' to fetch the default service account credentials.
64
+ stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
65
+ 'instance/service-accounts/default/token')
66
+ .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
67
+ status: 200,
68
+ headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
69
+ 'Content-Type' => 'application/json' })
70
+ end
71
+
72
+ def setup_ec2_metadata_stubs
73
+ # Stub the root, used for platform detection.
74
+ stub_request(:get, 'http://169.254.169.254')
75
+ .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
76
+
77
+ # Stub the identity document lookup made by the agent.
78
+ stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
79
+ 'instance-identity/document')
80
+ .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
81
+ headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
82
+ end
83
+
84
+ def setup_auth_stubs(base_url)
85
+ # Used when loading credentials from a JSON file.
86
+ stub_request(:post, base_url)
87
+ .with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
88
+ .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
89
+ status: 200,
90
+ headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
91
+ 'Content-Type' => 'application/json' })
92
+
93
+ stub_request(:post, base_url)
94
+ .with(body: hash_including(grant_type: 'refresh_token'))
95
+ .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
96
+ status: 200,
97
+ headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
98
+ 'Content-Type' => 'application/json' })
99
+ end
100
+
101
+ def setup_managed_vm_metadata_stubs
102
+ stub_metadata_request(
103
+ 'instance/attributes/',
104
+ "attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute")
105
+ stub_metadata_request('instance/attributes/gae_backend_name',
106
+ MANAGED_VM_BACKEND_NAME)
107
+ stub_metadata_request('instance/attributes/gae_backend_version',
108
+ MANAGED_VM_BACKEND_VERSION)
109
+ end
110
+
111
+ def setup_k8s_metadata_stubs(should_respond = true)
112
+ if should_respond
113
+ stub_metadata_request(
114
+ 'instance/attributes/',
115
+ "attribute1\ncluster-location\ncluster-name\nlast_attribute")
116
+ stub_metadata_request('instance/attributes/cluster-location',
117
+ K8S_LOCATION2)
118
+ stub_metadata_request('instance/attributes/cluster-name',
119
+ K8S_CLUSTER_NAME)
120
+ else
121
+ ['cluster-location', 'cluster-name'].each do |metadata_name|
122
+ stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
123
+ .to_return(status: 404,
124
+ body: 'The requested URL /computeMetadata/v1/instance/' \
125
+ "attributes/#{metadata_name} was not found on this" \
126
+ ' server.')
127
+ end
128
+ end
129
+ end
130
+
131
+ def setup_dataproc_metadata_stubs
132
+ stub_metadata_request(
133
+ 'instance/attributes/',
134
+ "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
135
+ stub_metadata_request('instance/attributes/dataproc-cluster-name',
136
+ DATAPROC_CLUSTER_NAME)
137
+ stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
138
+ DATAPROC_CLUSTER_UUID)
139
+ stub_metadata_request('instance/attributes/dataproc-region',
140
+ DATAPROC_REGION)
141
+ end
142
+
143
+ def clear_metrics
144
+ Prometheus::Client.registry.instance_variable_set('@metrics', {})
145
+ OpenCensus::Stats.ensure_recorder.clear_stats
146
+ end
147
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-google-cloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.4
4
+ version: 0.9.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stackdriver Agents Team
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-27 00:00:00.000000000 Z
11
+ date: 2020-07-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -86,28 +86,28 @@ dependencies:
86
86
  requirements:
87
87
  - - '='
88
88
  - !ruby/object:Gem::Version
89
- version: 3.9.0
89
+ version: 3.12.2
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - '='
95
95
  - !ruby/object:Gem::Version
96
- version: 3.9.0
96
+ version: 3.12.2
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: grpc
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - '='
102
102
  - !ruby/object:Gem::Version
103
- version: 1.22.0
103
+ version: 1.30.2
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - '='
109
109
  - !ruby/object:Gem::Version
110
- version: 1.22.0
110
+ version: 1.30.2
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: json
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -266,23 +266,30 @@ files:
266
266
  - README.rdoc
267
267
  - Rakefile
268
268
  - fluent-plugin-google-cloud.gemspec
269
+ - lib/fluent/plugin/common.rb
269
270
  - lib/fluent/plugin/filter_add_insert_ids.rb
271
+ - lib/fluent/plugin/filter_analyze_config.rb
270
272
  - lib/fluent/plugin/in_object_space_dump.rb
271
273
  - lib/fluent/plugin/monitoring.rb
272
274
  - lib/fluent/plugin/out_google_cloud.rb
273
275
  - lib/fluent/plugin/statusz.rb
274
276
  - test/helper.rb
277
+ - test/plugin/asserts.rb
275
278
  - test/plugin/base_test.rb
276
279
  - test/plugin/constants.rb
277
280
  - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
278
281
  - test/plugin/data/credentials.json
282
+ - test/plugin/data/google-fluentd-baseline.conf
283
+ - test/plugin/data/google-fluentd-custom.conf
279
284
  - test/plugin/data/iam-credentials.json
280
285
  - test/plugin/data/invalid_credentials.json
281
286
  - test/plugin/data/new-style-credentials.json
282
287
  - test/plugin/test_driver.rb
283
288
  - test/plugin/test_filter_add_insert_ids.rb
289
+ - test/plugin/test_filter_analyze_config.rb
284
290
  - test/plugin/test_out_google_cloud.rb
285
291
  - test/plugin/test_out_google_cloud_grpc.rb
292
+ - test/plugin/utils.rb
286
293
  homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
287
294
  licenses:
288
295
  - Apache-2.0
@@ -302,20 +309,25 @@ required_rubygems_version: !ruby/object:Gem::Requirement
302
309
  - !ruby/object:Gem::Version
303
310
  version: '0'
304
311
  requirements: []
305
- rubygems_version: 3.0.6
312
+ rubygems_version: 3.0.8
306
313
  signing_key:
307
314
  specification_version: 4
308
315
  summary: fluentd plugins for the Stackdriver Logging API
309
316
  test_files:
310
317
  - test/helper.rb
318
+ - test/plugin/asserts.rb
311
319
  - test/plugin/base_test.rb
312
320
  - test/plugin/constants.rb
313
321
  - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
314
322
  - test/plugin/data/credentials.json
323
+ - test/plugin/data/google-fluentd-baseline.conf
324
+ - test/plugin/data/google-fluentd-custom.conf
315
325
  - test/plugin/data/iam-credentials.json
316
326
  - test/plugin/data/invalid_credentials.json
317
327
  - test/plugin/data/new-style-credentials.json
318
328
  - test/plugin/test_driver.rb
319
329
  - test/plugin/test_filter_add_insert_ids.rb
330
+ - test/plugin/test_filter_analyze_config.rb
320
331
  - test/plugin/test_out_google_cloud.rb
321
332
  - test/plugin/test_out_google_cloud_grpc.rb
333
+ - test/plugin/utils.rb