fluent-plugin-vadimberezniker-gcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/LICENSE +201 -0
- data/README.rdoc +53 -0
- data/Rakefile +43 -0
- data/fluent-plugin-google-cloud.gemspec +43 -0
- data/fluent-plugin-vadimberezniker-gcp-0.13.2.gem +0 -0
- data/lib/fluent/plugin/common.rb +399 -0
- data/lib/fluent/plugin/filter_add_insert_ids.rb +86 -0
- data/lib/fluent/plugin/filter_analyze_config.rb +410 -0
- data/lib/fluent/plugin/in_object_space_dump.rb +62 -0
- data/lib/fluent/plugin/monitoring.rb +265 -0
- data/lib/fluent/plugin/out_google_cloud.rb +2209 -0
- data/lib/fluent/plugin/statusz.rb +124 -0
- data/test/helper.rb +46 -0
- data/test/plugin/asserts.rb +87 -0
- data/test/plugin/base_test.rb +2680 -0
- data/test/plugin/constants.rb +1114 -0
- data/test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12 +0 -0
- data/test/plugin/data/credentials.json +7 -0
- data/test/plugin/data/google-fluentd-baseline.conf +24 -0
- data/test/plugin/data/google-fluentd-custom.conf +40 -0
- data/test/plugin/data/iam-credentials.json +11 -0
- data/test/plugin/data/invalid_credentials.json +8 -0
- data/test/plugin/data/new-style-credentials.json +12 -0
- data/test/plugin/test_driver.rb +56 -0
- data/test/plugin/test_filter_add_insert_ids.rb +137 -0
- data/test/plugin/test_filter_analyze_config.rb +257 -0
- data/test/plugin/test_out_google_cloud.rb +465 -0
- data/test/plugin/test_out_google_cloud_grpc.rb +478 -0
- data/test/plugin/utils.rb +148 -0
- metadata +347 -0
@@ -0,0 +1,7 @@
|
|
1
|
+
{
|
2
|
+
"private_key_id": "cbedb7568906086cab57859bbfc1748749cc46c4",
|
3
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
|
4
|
+
"client_email": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl@developer.gserviceaccount.com",
|
5
|
+
"client_id": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl.apps.googleusercontent.com",
|
6
|
+
"type": "service_account"
|
7
|
+
}
|
@@ -0,0 +1,24 @@
|
|
1
|
+
<source>
|
2
|
+
@type syslog
|
3
|
+
port 514
|
4
|
+
protocol_type tcp
|
5
|
+
bind 127.0.0.1
|
6
|
+
tag syslog
|
7
|
+
</source>
|
8
|
+
|
9
|
+
<source>
|
10
|
+
@type tail
|
11
|
+
path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
|
12
|
+
pos_file /var/lib/google-fluentd/pos/apache-access.pos
|
13
|
+
tag apache-access
|
14
|
+
</source>
|
15
|
+
|
16
|
+
<filter **>
|
17
|
+
@type add_insert_ids
|
18
|
+
</filter>
|
19
|
+
|
20
|
+
<match **>
|
21
|
+
@type google_cloud
|
22
|
+
adjust_invalid_timestamps true
|
23
|
+
autoformat_stackdriver_trace true
|
24
|
+
</match>
|
@@ -0,0 +1,40 @@
|
|
1
|
+
<source>
|
2
|
+
@type syslog
|
3
|
+
port 514
|
4
|
+
protocol_type tcp
|
5
|
+
bind 127.0.0.1
|
6
|
+
tag syslog
|
7
|
+
</source>
|
8
|
+
|
9
|
+
<source>
|
10
|
+
@type tail
|
11
|
+
path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
|
12
|
+
pos_file /var/lib/google-fluentd/pos/apache-access.pos
|
13
|
+
tag apache-access
|
14
|
+
</source>
|
15
|
+
|
16
|
+
<filter **>
|
17
|
+
@type add_insert_ids
|
18
|
+
</filter>
|
19
|
+
|
20
|
+
<match **>
|
21
|
+
@type google_cloud
|
22
|
+
adjust_invalid_timestamps true
|
23
|
+
autoformat_stackdriver_trace false
|
24
|
+
coerce_to_utf8 true
|
25
|
+
</match>
|
26
|
+
|
27
|
+
<filter **>
|
28
|
+
@type some_custom_filter
|
29
|
+
</filter>
|
30
|
+
|
31
|
+
<filter **>
|
32
|
+
@type record_transformer
|
33
|
+
<record>
|
34
|
+
host_param "#{Socket.gethostname}"
|
35
|
+
</record>
|
36
|
+
</filter>
|
37
|
+
|
38
|
+
<match "app.#{ENV['FLUENTD_TAG']}">
|
39
|
+
@type stdout
|
40
|
+
</match>
|
@@ -0,0 +1,11 @@
|
|
1
|
+
{
|
2
|
+
"type": "service_account",
|
3
|
+
"private_key_id": "5985985bcdfe958895bd8d76456fe90d8484789d",
|
4
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
|
5
|
+
"client_email": "account-name@fluent-test-project.iam.gserviceaccount.com",
|
6
|
+
"client_id": "275859789789367827863",
|
7
|
+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
8
|
+
"token_uri": "https://accounts.google.com/o/oauth2/token",
|
9
|
+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
|
10
|
+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/account-name%40fluent-test-project.iam.gserviceaccount.com"
|
11
|
+
}
|
@@ -0,0 +1,8 @@
|
|
1
|
+
{
|
2
|
+
"private_key_id": "cbedb7568906086cab57859bbfc1748749cc46c4",
|
3
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nCeci n'est pas une cle\n-----END PRIVATE KEY-----\n",
|
4
|
+
"client_email": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl@developer.gserviceaccount.com",
|
5
|
+
"client_id": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl.apps.googleusercontent.com",
|
6
|
+
"type": "service_account"
|
7
|
+
}
|
8
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
{
|
2
|
+
"type": "service_account",
|
3
|
+
"private_key_id": "5985985bcdfe958895bd8d76456fe90d8484789d",
|
4
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
|
5
|
+
"client_email": "account-name@fluent-test-project-1.iam.gserviceaccount.com",
|
6
|
+
"client_id": "275859789789367827863",
|
7
|
+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
8
|
+
"token_uri": "https://accounts.google.com/o/oauth2/token",
|
9
|
+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
|
10
|
+
"project_id": "fluent-test-project",
|
11
|
+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/account-name%40fluent-test-project.iam.gserviceaccount.com"
|
12
|
+
}
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# Copyright 2018 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/engine'
|
16
|
+
require 'fluent/event'
|
17
|
+
require 'fluent/test/input_test'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Test
|
21
|
+
# Similar to the standard BufferedOutputTestDriver, but allows multiple tags
|
22
|
+
# to exist in one chunk.
|
23
|
+
class MultiTagBufferedOutputTestDriver < InputTestDriver
|
24
|
+
def initialize(klass, &block)
|
25
|
+
super(klass, &block)
|
26
|
+
@entries = []
|
27
|
+
end
|
28
|
+
|
29
|
+
def emit(tag, record, time = Engine.now)
|
30
|
+
es = ArrayEventStream.new([[time, record]])
|
31
|
+
data = @instance.format_stream(tag, es)
|
32
|
+
@entries << data
|
33
|
+
self
|
34
|
+
end
|
35
|
+
|
36
|
+
def run(num_waits = 10)
|
37
|
+
result = nil
|
38
|
+
super(num_waits) do
|
39
|
+
chunk = @instance.buffer.generate_chunk(
|
40
|
+
@instance.metadata(nil, nil, nil)
|
41
|
+
).staged!
|
42
|
+
@entries.each do |entry|
|
43
|
+
chunk.concat(entry, 1)
|
44
|
+
end
|
45
|
+
|
46
|
+
begin
|
47
|
+
result = @instance.write(chunk)
|
48
|
+
ensure
|
49
|
+
chunk.purge
|
50
|
+
end
|
51
|
+
end
|
52
|
+
result
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
# Copyright 2018 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require_relative '../helper'
|
16
|
+
|
17
|
+
require 'fluent/test/driver/filter'
|
18
|
+
require 'fluent/plugin/filter_add_insert_ids'
|
19
|
+
|
20
|
+
# Unit tests for filter_add_insert_ids plugin.
|
21
|
+
class FilterAddInsertIdsTest < Test::Unit::TestCase
|
22
|
+
include Fluent::AddInsertIdsFilter::ConfigConstants
|
23
|
+
|
24
|
+
CUSTOM_INSERT_ID_KEY = 'custom_insert_id_key'.freeze
|
25
|
+
INSERT_ID = 'aeyr82r92h249gh9h'.freeze
|
26
|
+
TEST_MESSAGE = 'test message for add_insert_ids plugin.'.freeze
|
27
|
+
APPLICATION_DEFAULT_CONFIG = ''.freeze
|
28
|
+
INSERT_ID_KEY_CONFIG = %(
|
29
|
+
insert_id_key #{CUSTOM_INSERT_ID_KEY}
|
30
|
+
).freeze
|
31
|
+
|
32
|
+
def setup
|
33
|
+
Fluent::Test.setup
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_configure_insert_id_key
|
37
|
+
{
|
38
|
+
APPLICATION_DEFAULT_CONFIG => DEFAULT_INSERT_ID_KEY,
|
39
|
+
INSERT_ID_KEY_CONFIG => CUSTOM_INSERT_ID_KEY
|
40
|
+
}.each do |config, insert_id_key|
|
41
|
+
d = create_driver(config)
|
42
|
+
assert_equal insert_id_key, d.instance.insert_id_key
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_add_insert_ids
|
47
|
+
total_entry_count = 1000
|
48
|
+
d = create_driver
|
49
|
+
d.run do
|
50
|
+
total_entry_count.times do |index|
|
51
|
+
d.emit(log_entry(index))
|
52
|
+
end
|
53
|
+
end
|
54
|
+
filtered_events = d.filtered_as_array
|
55
|
+
|
56
|
+
assert_equal total_entry_count, filtered_events.size,
|
57
|
+
"#{total_entry_count} log entries after filtering is" \
|
58
|
+
" expected. Only #{filtered_events.size} are detected."
|
59
|
+
# The expected insertId will be assigned as we scan the first log entry.
|
60
|
+
expected_insert_id = nil
|
61
|
+
unique_insert_ids = Set.new
|
62
|
+
filtered_events.each_with_index do |event, index|
|
63
|
+
assert_equal 3, event.size, "Index #{index} failed. Log event should" \
|
64
|
+
' include 3 elements: tag, time and record.'
|
65
|
+
record = event[2]
|
66
|
+
assert_true record.is_a?(Hash), "Index #{index} failed. Log record" \
|
67
|
+
" #{record} should be a hash."
|
68
|
+
assert_equal index, record['id'], "Index #{index} failed. Log entries" \
|
69
|
+
' should come in order.'
|
70
|
+
assert_equal TEST_MESSAGE, record['message'], "Index #{index} failed."
|
71
|
+
|
72
|
+
# Get the first insertID.
|
73
|
+
expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index.zero?
|
74
|
+
insert_id = record[DEFAULT_INSERT_ID_KEY]
|
75
|
+
assert_equal expected_insert_id, insert_id, "Index #{index} failed."
|
76
|
+
expected_insert_id = expected_insert_id.next
|
77
|
+
assert_true insert_id < expected_insert_id,
|
78
|
+
"Index #{index} failed. #{insert_id}" \
|
79
|
+
" < #{expected_insert_id} is false."
|
80
|
+
unique_insert_ids << insert_id
|
81
|
+
end
|
82
|
+
assert_equal total_entry_count, unique_insert_ids.size,
|
83
|
+
"Expected #{total_entry_count} unique insertIds." \
|
84
|
+
" Only #{unique_insert_ids.size} found."
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_insert_ids_not_added_if_present
|
88
|
+
log_entry_with_empty_insert_id = log_entry(0).merge(
|
89
|
+
DEFAULT_INSERT_ID_KEY => ''
|
90
|
+
)
|
91
|
+
{
|
92
|
+
log_entry(0).merge(DEFAULT_INSERT_ID_KEY => INSERT_ID) => true,
|
93
|
+
# Still generate insertId if it's an empty string
|
94
|
+
log_entry_with_empty_insert_id => false
|
95
|
+
}.each do |test_data|
|
96
|
+
input_log_entry, retain_original_insert_id = test_data
|
97
|
+
# Make a copy because the log entry gets modified by the filter plugin.
|
98
|
+
log_entry = input_log_entry.dup
|
99
|
+
d = create_driver
|
100
|
+
d.run do
|
101
|
+
d.emit(log_entry)
|
102
|
+
end
|
103
|
+
filtered_events = d.filtered_as_array
|
104
|
+
|
105
|
+
assert_equal 1, filtered_events.size, 'Exact 1 log entry after' \
|
106
|
+
" filtering is expected. Test data: #{test_data}."
|
107
|
+
event = filtered_events[0]
|
108
|
+
assert_equal 3, event.size, 'Log event should include 3 elements: tag,' \
|
109
|
+
" time and record. Test data: #{test_data}."
|
110
|
+
record = event[2]
|
111
|
+
assert_true record.is_a?(Hash), "Log record #{record} should be a hash." \
|
112
|
+
" Test data: #{test_data}."
|
113
|
+
assert_equal 0, record['id'], "Test data: #{test_data}."
|
114
|
+
assert_equal TEST_MESSAGE, record['message'], "Test data: #{test_data}."
|
115
|
+
insert_id = record[DEFAULT_INSERT_ID_KEY]
|
116
|
+
assert_false insert_id.to_s.empty?, 'Insert ID should not be empty.' \
|
117
|
+
" Test data: #{test_data}."
|
118
|
+
assert_equal retain_original_insert_id,
|
119
|
+
input_log_entry[DEFAULT_INSERT_ID_KEY] == insert_id,
|
120
|
+
"Input value is #{input_log_entry[DEFAULT_INSERT_ID_KEY]}." \
|
121
|
+
" Output value is #{insert_id}. Test data: #{test_data}."
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
126
|
+
Fluent::Test::FilterTestDriver.new(
|
127
|
+
Fluent::AddInsertIdsFilter
|
128
|
+
).configure(conf, true)
|
129
|
+
end
|
130
|
+
|
131
|
+
def log_entry(index)
|
132
|
+
{
|
133
|
+
'id' => index,
|
134
|
+
'message' => TEST_MESSAGE
|
135
|
+
}
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,257 @@
|
|
1
|
+
# Copyright 2020 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require_relative '../helper'
|
16
|
+
require_relative 'asserts'
|
17
|
+
require_relative 'constants'
|
18
|
+
require_relative 'utils'
|
19
|
+
|
20
|
+
require 'fluent/test/driver/filter'
|
21
|
+
require 'fluent/plugin/filter_analyze_config'
|
22
|
+
|
23
|
+
# Unit tests for filter_analyze_config plugin.
|
24
|
+
class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
25
|
+
include Asserts
|
26
|
+
include Constants
|
27
|
+
include Fluent::AnalyzeConfigFilter::Constants
|
28
|
+
include Utils
|
29
|
+
|
30
|
+
APPLICATION_DEFAULT_CONFIG = ''.freeze
|
31
|
+
|
32
|
+
def setup
|
33
|
+
Fluent::Test.setup
|
34
|
+
delete_env_vars
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_config_file_does_not_exist
|
38
|
+
# By default, the FilterTestDriver.new does not set up a config file at:
|
39
|
+
# /etc/google-fluentd/google-fluentd.conf. The plugin should still proceed.
|
40
|
+
create_driver
|
41
|
+
# No exceptions were thrown.
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_analyze_config
|
45
|
+
setup_auth_stubs('https://oauth2.googleapis.com/token')
|
46
|
+
setup_gce_metadata_stubs
|
47
|
+
[
|
48
|
+
[CONFIG_ANALYZE_CONFIG_PROMETHEUS,
|
49
|
+
method(:assert_prometheus_metric_value)],
|
50
|
+
[CONFIG_ANALYZE_CONFIG_OPENCENSUS,
|
51
|
+
method(:assert_opencensus_metric_value)]
|
52
|
+
].each do |config, assert_metric_value|
|
53
|
+
clear_metrics
|
54
|
+
d = create_driver(config)
|
55
|
+
|
56
|
+
# Default plugins, with default config.
|
57
|
+
assert_metric_value.call(
|
58
|
+
:enabled_plugins,
|
59
|
+
1,
|
60
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
61
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
62
|
+
d,
|
63
|
+
plugin_name: 'source/syslog/tcp',
|
64
|
+
is_default_plugin: true,
|
65
|
+
has_default_config: true,
|
66
|
+
has_ruby_snippet: false
|
67
|
+
)
|
68
|
+
assert_metric_value.call(
|
69
|
+
:enabled_plugins,
|
70
|
+
1,
|
71
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
72
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
73
|
+
d,
|
74
|
+
plugin_name: 'source/tail/apache-access',
|
75
|
+
is_default_plugin: true,
|
76
|
+
has_default_config: true,
|
77
|
+
has_ruby_snippet: false
|
78
|
+
)
|
79
|
+
assert_metric_value.call(
|
80
|
+
:enabled_plugins,
|
81
|
+
1,
|
82
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
83
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
84
|
+
d,
|
85
|
+
plugin_name: 'filter/add_insert_ids',
|
86
|
+
is_default_plugin: true,
|
87
|
+
has_default_config: true,
|
88
|
+
has_ruby_snippet: false
|
89
|
+
)
|
90
|
+
|
91
|
+
# Default plugins, with custom config.
|
92
|
+
assert_metric_value.call(
|
93
|
+
:enabled_plugins,
|
94
|
+
1,
|
95
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
96
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
97
|
+
d,
|
98
|
+
plugin_name: 'match/google_cloud',
|
99
|
+
is_default_plugin: true,
|
100
|
+
has_default_config: false,
|
101
|
+
has_ruby_snippet: false
|
102
|
+
)
|
103
|
+
|
104
|
+
# Custom plugins, some with embedded Ruby.
|
105
|
+
assert_metric_value.call(
|
106
|
+
:enabled_plugins,
|
107
|
+
1,
|
108
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
109
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
110
|
+
d,
|
111
|
+
plugin_name: 'filter',
|
112
|
+
is_default_plugin: false,
|
113
|
+
has_default_config: false,
|
114
|
+
has_ruby_snippet: false
|
115
|
+
)
|
116
|
+
assert_metric_value.call(
|
117
|
+
:enabled_plugins,
|
118
|
+
1,
|
119
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
120
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
121
|
+
d,
|
122
|
+
plugin_name: 'filter/record_transformer',
|
123
|
+
is_default_plugin: false,
|
124
|
+
has_default_config: false,
|
125
|
+
has_ruby_snippet: true
|
126
|
+
)
|
127
|
+
assert_metric_value.call(
|
128
|
+
:enabled_plugins,
|
129
|
+
1,
|
130
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
131
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
132
|
+
d,
|
133
|
+
plugin_name: 'match/stdout',
|
134
|
+
is_default_plugin: false,
|
135
|
+
has_default_config: false,
|
136
|
+
has_ruby_snippet: true
|
137
|
+
)
|
138
|
+
|
139
|
+
# For out_google_cloud, 3 params are present.
|
140
|
+
assert_metric_value.call(
|
141
|
+
:plugin_config,
|
142
|
+
1,
|
143
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
144
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
145
|
+
d,
|
146
|
+
plugin_name: 'google_cloud',
|
147
|
+
param: 'adjust_invalid_timestamps',
|
148
|
+
is_present: true,
|
149
|
+
has_default_config: true
|
150
|
+
)
|
151
|
+
assert_metric_value.call(
|
152
|
+
:plugin_config,
|
153
|
+
1,
|
154
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
155
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
156
|
+
d,
|
157
|
+
plugin_name: 'google_cloud',
|
158
|
+
param: 'autoformat_stackdriver_trace',
|
159
|
+
is_present: true,
|
160
|
+
has_default_config: false
|
161
|
+
)
|
162
|
+
assert_metric_value.call(
|
163
|
+
:plugin_config,
|
164
|
+
1,
|
165
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
166
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
167
|
+
d,
|
168
|
+
plugin_name: 'google_cloud',
|
169
|
+
param: 'coerce_to_utf8',
|
170
|
+
is_present: true,
|
171
|
+
has_default_config: false
|
172
|
+
)
|
173
|
+
# The remaining "google_cloud" params are not present.
|
174
|
+
# The are no params for "detect_exceptions".
|
175
|
+
%w[
|
176
|
+
auth_method
|
177
|
+
detect_json
|
178
|
+
enable_monitoring
|
179
|
+
gcm_service_address
|
180
|
+
grpc_compression_algorithm
|
181
|
+
http_request_key
|
182
|
+
insert_id_key
|
183
|
+
label_map
|
184
|
+
labels
|
185
|
+
labels_key
|
186
|
+
logging_api_url
|
187
|
+
monitoring_type
|
188
|
+
non_utf8_replacement_string
|
189
|
+
operation_key
|
190
|
+
private_key_email
|
191
|
+
private_key_passphrase
|
192
|
+
private_key_path
|
193
|
+
project_id
|
194
|
+
source_location_key
|
195
|
+
span_id_key
|
196
|
+
statusz_port
|
197
|
+
trace_key
|
198
|
+
trace_sampled_key
|
199
|
+
use_grpc
|
200
|
+
use_metadata_service
|
201
|
+
vm_id
|
202
|
+
vm_name
|
203
|
+
zone
|
204
|
+
].each do |p|
|
205
|
+
assert_metric_value.call(
|
206
|
+
:plugin_config,
|
207
|
+
1,
|
208
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
209
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
210
|
+
d,
|
211
|
+
plugin_name: 'google_cloud',
|
212
|
+
param: p,
|
213
|
+
is_present: false,
|
214
|
+
has_default_config: false
|
215
|
+
)
|
216
|
+
end
|
217
|
+
|
218
|
+
# We also export values for the bools.
|
219
|
+
assert_metric_value.call(
|
220
|
+
:config_bool_values,
|
221
|
+
1,
|
222
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
223
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
224
|
+
d,
|
225
|
+
plugin_name: 'google_cloud',
|
226
|
+
param: 'adjust_invalid_timestamps',
|
227
|
+
value: true
|
228
|
+
)
|
229
|
+
assert_metric_value.call(
|
230
|
+
:config_bool_values,
|
231
|
+
1,
|
232
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
233
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
234
|
+
d,
|
235
|
+
plugin_name: 'google_cloud',
|
236
|
+
param: 'autoformat_stackdriver_trace',
|
237
|
+
value: false
|
238
|
+
)
|
239
|
+
assert_metric_value.call(
|
240
|
+
:config_bool_values,
|
241
|
+
1,
|
242
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
243
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
244
|
+
d,
|
245
|
+
plugin_name: 'google_cloud',
|
246
|
+
param: 'coerce_to_utf8',
|
247
|
+
value: true
|
248
|
+
)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
253
|
+
Fluent::Test::FilterTestDriver.new(
|
254
|
+
Fluent::AnalyzeConfigFilter
|
255
|
+
).configure(conf, true)
|
256
|
+
end
|
257
|
+
end
|