fluent-plugin-vadimberezniker-gcp 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/LICENSE +201 -0
- data/README.rdoc +53 -0
- data/Rakefile +43 -0
- data/fluent-plugin-google-cloud.gemspec +43 -0
- data/fluent-plugin-vadimberezniker-gcp-0.13.2.gem +0 -0
- data/lib/fluent/plugin/common.rb +399 -0
- data/lib/fluent/plugin/filter_add_insert_ids.rb +86 -0
- data/lib/fluent/plugin/filter_analyze_config.rb +410 -0
- data/lib/fluent/plugin/in_object_space_dump.rb +62 -0
- data/lib/fluent/plugin/monitoring.rb +265 -0
- data/lib/fluent/plugin/out_google_cloud.rb +2209 -0
- data/lib/fluent/plugin/statusz.rb +124 -0
- data/test/helper.rb +46 -0
- data/test/plugin/asserts.rb +87 -0
- data/test/plugin/base_test.rb +2680 -0
- data/test/plugin/constants.rb +1114 -0
- data/test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12 +0 -0
- data/test/plugin/data/credentials.json +7 -0
- data/test/plugin/data/google-fluentd-baseline.conf +24 -0
- data/test/plugin/data/google-fluentd-custom.conf +40 -0
- data/test/plugin/data/iam-credentials.json +11 -0
- data/test/plugin/data/invalid_credentials.json +8 -0
- data/test/plugin/data/new-style-credentials.json +12 -0
- data/test/plugin/test_driver.rb +56 -0
- data/test/plugin/test_filter_add_insert_ids.rb +137 -0
- data/test/plugin/test_filter_analyze_config.rb +257 -0
- data/test/plugin/test_out_google_cloud.rb +465 -0
- data/test/plugin/test_out_google_cloud_grpc.rb +478 -0
- data/test/plugin/utils.rb +148 -0
- metadata +347 -0
@@ -0,0 +1,7 @@
|
|
1
|
+
{
|
2
|
+
"private_key_id": "cbedb7568906086cab57859bbfc1748749cc46c4",
|
3
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
|
4
|
+
"client_email": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl@developer.gserviceaccount.com",
|
5
|
+
"client_id": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl.apps.googleusercontent.com",
|
6
|
+
"type": "service_account"
|
7
|
+
}
|
@@ -0,0 +1,24 @@
|
|
1
|
+
<source>
|
2
|
+
@type syslog
|
3
|
+
port 514
|
4
|
+
protocol_type tcp
|
5
|
+
bind 127.0.0.1
|
6
|
+
tag syslog
|
7
|
+
</source>
|
8
|
+
|
9
|
+
<source>
|
10
|
+
@type tail
|
11
|
+
path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
|
12
|
+
pos_file /var/lib/google-fluentd/pos/apache-access.pos
|
13
|
+
tag apache-access
|
14
|
+
</source>
|
15
|
+
|
16
|
+
<filter **>
|
17
|
+
@type add_insert_ids
|
18
|
+
</filter>
|
19
|
+
|
20
|
+
<match **>
|
21
|
+
@type google_cloud
|
22
|
+
adjust_invalid_timestamps true
|
23
|
+
autoformat_stackdriver_trace true
|
24
|
+
</match>
|
@@ -0,0 +1,40 @@
|
|
1
|
+
<source>
|
2
|
+
@type syslog
|
3
|
+
port 514
|
4
|
+
protocol_type tcp
|
5
|
+
bind 127.0.0.1
|
6
|
+
tag syslog
|
7
|
+
</source>
|
8
|
+
|
9
|
+
<source>
|
10
|
+
@type tail
|
11
|
+
path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
|
12
|
+
pos_file /var/lib/google-fluentd/pos/apache-access.pos
|
13
|
+
tag apache-access
|
14
|
+
</source>
|
15
|
+
|
16
|
+
<filter **>
|
17
|
+
@type add_insert_ids
|
18
|
+
</filter>
|
19
|
+
|
20
|
+
<match **>
|
21
|
+
@type google_cloud
|
22
|
+
adjust_invalid_timestamps true
|
23
|
+
autoformat_stackdriver_trace false
|
24
|
+
coerce_to_utf8 true
|
25
|
+
</match>
|
26
|
+
|
27
|
+
<filter **>
|
28
|
+
@type some_custom_filter
|
29
|
+
</filter>
|
30
|
+
|
31
|
+
<filter **>
|
32
|
+
@type record_transformer
|
33
|
+
<record>
|
34
|
+
host_param "#{Socket.gethostname}"
|
35
|
+
</record>
|
36
|
+
</filter>
|
37
|
+
|
38
|
+
<match "app.#{ENV['FLUENTD_TAG']}">
|
39
|
+
@type stdout
|
40
|
+
</match>
|
@@ -0,0 +1,11 @@
|
|
1
|
+
{
|
2
|
+
"type": "service_account",
|
3
|
+
"private_key_id": "5985985bcdfe958895bd8d76456fe90d8484789d",
|
4
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
|
5
|
+
"client_email": "account-name@fluent-test-project.iam.gserviceaccount.com",
|
6
|
+
"client_id": "275859789789367827863",
|
7
|
+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
8
|
+
"token_uri": "https://accounts.google.com/o/oauth2/token",
|
9
|
+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
|
10
|
+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/account-name%40fluent-test-project.iam.gserviceaccount.com"
|
11
|
+
}
|
@@ -0,0 +1,8 @@
|
|
1
|
+
{
|
2
|
+
"private_key_id": "cbedb7568906086cab57859bbfc1748749cc46c4",
|
3
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nCeci n'est pas une cle\n-----END PRIVATE KEY-----\n",
|
4
|
+
"client_email": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl@developer.gserviceaccount.com",
|
5
|
+
"client_id": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl.apps.googleusercontent.com",
|
6
|
+
"type": "service_account"
|
7
|
+
}
|
8
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
{
|
2
|
+
"type": "service_account",
|
3
|
+
"private_key_id": "5985985bcdfe958895bd8d76456fe90d8484789d",
|
4
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
|
5
|
+
"client_email": "account-name@fluent-test-project-1.iam.gserviceaccount.com",
|
6
|
+
"client_id": "275859789789367827863",
|
7
|
+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
8
|
+
"token_uri": "https://accounts.google.com/o/oauth2/token",
|
9
|
+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
|
10
|
+
"project_id": "fluent-test-project",
|
11
|
+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/account-name%40fluent-test-project.iam.gserviceaccount.com"
|
12
|
+
}
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# Copyright 2018 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/engine'
|
16
|
+
require 'fluent/event'
|
17
|
+
require 'fluent/test/input_test'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Test
|
21
|
+
# Similar to the standard BufferedOutputTestDriver, but allows multiple tags
|
22
|
+
# to exist in one chunk.
|
23
|
+
class MultiTagBufferedOutputTestDriver < InputTestDriver
|
24
|
+
def initialize(klass, &block)
|
25
|
+
super(klass, &block)
|
26
|
+
@entries = []
|
27
|
+
end
|
28
|
+
|
29
|
+
def emit(tag, record, time = Engine.now)
|
30
|
+
es = ArrayEventStream.new([[time, record]])
|
31
|
+
data = @instance.format_stream(tag, es)
|
32
|
+
@entries << data
|
33
|
+
self
|
34
|
+
end
|
35
|
+
|
36
|
+
def run(num_waits = 10)
|
37
|
+
result = nil
|
38
|
+
super(num_waits) do
|
39
|
+
chunk = @instance.buffer.generate_chunk(
|
40
|
+
@instance.metadata(nil, nil, nil)
|
41
|
+
).staged!
|
42
|
+
@entries.each do |entry|
|
43
|
+
chunk.concat(entry, 1)
|
44
|
+
end
|
45
|
+
|
46
|
+
begin
|
47
|
+
result = @instance.write(chunk)
|
48
|
+
ensure
|
49
|
+
chunk.purge
|
50
|
+
end
|
51
|
+
end
|
52
|
+
result
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
# Copyright 2018 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require_relative '../helper'
|
16
|
+
|
17
|
+
require 'fluent/test/driver/filter'
|
18
|
+
require 'fluent/plugin/filter_add_insert_ids'
|
19
|
+
|
20
|
+
# Unit tests for filter_add_insert_ids plugin.
|
21
|
+
class FilterAddInsertIdsTest < Test::Unit::TestCase
|
22
|
+
include Fluent::AddInsertIdsFilter::ConfigConstants
|
23
|
+
|
24
|
+
CUSTOM_INSERT_ID_KEY = 'custom_insert_id_key'.freeze
|
25
|
+
INSERT_ID = 'aeyr82r92h249gh9h'.freeze
|
26
|
+
TEST_MESSAGE = 'test message for add_insert_ids plugin.'.freeze
|
27
|
+
APPLICATION_DEFAULT_CONFIG = ''.freeze
|
28
|
+
INSERT_ID_KEY_CONFIG = %(
|
29
|
+
insert_id_key #{CUSTOM_INSERT_ID_KEY}
|
30
|
+
).freeze
|
31
|
+
|
32
|
+
def setup
|
33
|
+
Fluent::Test.setup
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_configure_insert_id_key
|
37
|
+
{
|
38
|
+
APPLICATION_DEFAULT_CONFIG => DEFAULT_INSERT_ID_KEY,
|
39
|
+
INSERT_ID_KEY_CONFIG => CUSTOM_INSERT_ID_KEY
|
40
|
+
}.each do |config, insert_id_key|
|
41
|
+
d = create_driver(config)
|
42
|
+
assert_equal insert_id_key, d.instance.insert_id_key
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_add_insert_ids
|
47
|
+
total_entry_count = 1000
|
48
|
+
d = create_driver
|
49
|
+
d.run do
|
50
|
+
total_entry_count.times do |index|
|
51
|
+
d.emit(log_entry(index))
|
52
|
+
end
|
53
|
+
end
|
54
|
+
filtered_events = d.filtered_as_array
|
55
|
+
|
56
|
+
assert_equal total_entry_count, filtered_events.size,
|
57
|
+
"#{total_entry_count} log entries after filtering is" \
|
58
|
+
" expected. Only #{filtered_events.size} are detected."
|
59
|
+
# The expected insertId will be assigned as we scan the first log entry.
|
60
|
+
expected_insert_id = nil
|
61
|
+
unique_insert_ids = Set.new
|
62
|
+
filtered_events.each_with_index do |event, index|
|
63
|
+
assert_equal 3, event.size, "Index #{index} failed. Log event should" \
|
64
|
+
' include 3 elements: tag, time and record.'
|
65
|
+
record = event[2]
|
66
|
+
assert_true record.is_a?(Hash), "Index #{index} failed. Log record" \
|
67
|
+
" #{record} should be a hash."
|
68
|
+
assert_equal index, record['id'], "Index #{index} failed. Log entries" \
|
69
|
+
' should come in order.'
|
70
|
+
assert_equal TEST_MESSAGE, record['message'], "Index #{index} failed."
|
71
|
+
|
72
|
+
# Get the first insertID.
|
73
|
+
expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index.zero?
|
74
|
+
insert_id = record[DEFAULT_INSERT_ID_KEY]
|
75
|
+
assert_equal expected_insert_id, insert_id, "Index #{index} failed."
|
76
|
+
expected_insert_id = expected_insert_id.next
|
77
|
+
assert_true insert_id < expected_insert_id,
|
78
|
+
"Index #{index} failed. #{insert_id}" \
|
79
|
+
" < #{expected_insert_id} is false."
|
80
|
+
unique_insert_ids << insert_id
|
81
|
+
end
|
82
|
+
assert_equal total_entry_count, unique_insert_ids.size,
|
83
|
+
"Expected #{total_entry_count} unique insertIds." \
|
84
|
+
" Only #{unique_insert_ids.size} found."
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_insert_ids_not_added_if_present
|
88
|
+
log_entry_with_empty_insert_id = log_entry(0).merge(
|
89
|
+
DEFAULT_INSERT_ID_KEY => ''
|
90
|
+
)
|
91
|
+
{
|
92
|
+
log_entry(0).merge(DEFAULT_INSERT_ID_KEY => INSERT_ID) => true,
|
93
|
+
# Still generate insertId if it's an empty string
|
94
|
+
log_entry_with_empty_insert_id => false
|
95
|
+
}.each do |test_data|
|
96
|
+
input_log_entry, retain_original_insert_id = test_data
|
97
|
+
# Make a copy because the log entry gets modified by the filter plugin.
|
98
|
+
log_entry = input_log_entry.dup
|
99
|
+
d = create_driver
|
100
|
+
d.run do
|
101
|
+
d.emit(log_entry)
|
102
|
+
end
|
103
|
+
filtered_events = d.filtered_as_array
|
104
|
+
|
105
|
+
assert_equal 1, filtered_events.size, 'Exact 1 log entry after' \
|
106
|
+
" filtering is expected. Test data: #{test_data}."
|
107
|
+
event = filtered_events[0]
|
108
|
+
assert_equal 3, event.size, 'Log event should include 3 elements: tag,' \
|
109
|
+
" time and record. Test data: #{test_data}."
|
110
|
+
record = event[2]
|
111
|
+
assert_true record.is_a?(Hash), "Log record #{record} should be a hash." \
|
112
|
+
" Test data: #{test_data}."
|
113
|
+
assert_equal 0, record['id'], "Test data: #{test_data}."
|
114
|
+
assert_equal TEST_MESSAGE, record['message'], "Test data: #{test_data}."
|
115
|
+
insert_id = record[DEFAULT_INSERT_ID_KEY]
|
116
|
+
assert_false insert_id.to_s.empty?, 'Insert ID should not be empty.' \
|
117
|
+
" Test data: #{test_data}."
|
118
|
+
assert_equal retain_original_insert_id,
|
119
|
+
input_log_entry[DEFAULT_INSERT_ID_KEY] == insert_id,
|
120
|
+
"Input value is #{input_log_entry[DEFAULT_INSERT_ID_KEY]}." \
|
121
|
+
" Output value is #{insert_id}. Test data: #{test_data}."
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
126
|
+
Fluent::Test::FilterTestDriver.new(
|
127
|
+
Fluent::AddInsertIdsFilter
|
128
|
+
).configure(conf, true)
|
129
|
+
end
|
130
|
+
|
131
|
+
def log_entry(index)
|
132
|
+
{
|
133
|
+
'id' => index,
|
134
|
+
'message' => TEST_MESSAGE
|
135
|
+
}
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,257 @@
|
|
1
|
+
# Copyright 2020 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require_relative '../helper'
|
16
|
+
require_relative 'asserts'
|
17
|
+
require_relative 'constants'
|
18
|
+
require_relative 'utils'
|
19
|
+
|
20
|
+
require 'fluent/test/driver/filter'
|
21
|
+
require 'fluent/plugin/filter_analyze_config'
|
22
|
+
|
23
|
+
# Unit tests for filter_analyze_config plugin.
|
24
|
+
class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
25
|
+
include Asserts
|
26
|
+
include Constants
|
27
|
+
include Fluent::AnalyzeConfigFilter::Constants
|
28
|
+
include Utils
|
29
|
+
|
30
|
+
APPLICATION_DEFAULT_CONFIG = ''.freeze
|
31
|
+
|
32
|
+
def setup
|
33
|
+
Fluent::Test.setup
|
34
|
+
delete_env_vars
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_config_file_does_not_exist
|
38
|
+
# By default, the FilterTestDriver.new does not set up a config file at:
|
39
|
+
# /etc/google-fluentd/google-fluentd.conf. The plugin should still proceed.
|
40
|
+
create_driver
|
41
|
+
# No exceptions were thrown.
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_analyze_config
|
45
|
+
setup_auth_stubs('https://oauth2.googleapis.com/token')
|
46
|
+
setup_gce_metadata_stubs
|
47
|
+
[
|
48
|
+
[CONFIG_ANALYZE_CONFIG_PROMETHEUS,
|
49
|
+
method(:assert_prometheus_metric_value)],
|
50
|
+
[CONFIG_ANALYZE_CONFIG_OPENCENSUS,
|
51
|
+
method(:assert_opencensus_metric_value)]
|
52
|
+
].each do |config, assert_metric_value|
|
53
|
+
clear_metrics
|
54
|
+
d = create_driver(config)
|
55
|
+
|
56
|
+
# Default plugins, with default config.
|
57
|
+
assert_metric_value.call(
|
58
|
+
:enabled_plugins,
|
59
|
+
1,
|
60
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
61
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
62
|
+
d,
|
63
|
+
plugin_name: 'source/syslog/tcp',
|
64
|
+
is_default_plugin: true,
|
65
|
+
has_default_config: true,
|
66
|
+
has_ruby_snippet: false
|
67
|
+
)
|
68
|
+
assert_metric_value.call(
|
69
|
+
:enabled_plugins,
|
70
|
+
1,
|
71
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
72
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
73
|
+
d,
|
74
|
+
plugin_name: 'source/tail/apache-access',
|
75
|
+
is_default_plugin: true,
|
76
|
+
has_default_config: true,
|
77
|
+
has_ruby_snippet: false
|
78
|
+
)
|
79
|
+
assert_metric_value.call(
|
80
|
+
:enabled_plugins,
|
81
|
+
1,
|
82
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
83
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
84
|
+
d,
|
85
|
+
plugin_name: 'filter/add_insert_ids',
|
86
|
+
is_default_plugin: true,
|
87
|
+
has_default_config: true,
|
88
|
+
has_ruby_snippet: false
|
89
|
+
)
|
90
|
+
|
91
|
+
# Default plugins, with custom config.
|
92
|
+
assert_metric_value.call(
|
93
|
+
:enabled_plugins,
|
94
|
+
1,
|
95
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
96
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
97
|
+
d,
|
98
|
+
plugin_name: 'match/google_cloud',
|
99
|
+
is_default_plugin: true,
|
100
|
+
has_default_config: false,
|
101
|
+
has_ruby_snippet: false
|
102
|
+
)
|
103
|
+
|
104
|
+
# Custom plugins, some with embedded Ruby.
|
105
|
+
assert_metric_value.call(
|
106
|
+
:enabled_plugins,
|
107
|
+
1,
|
108
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
109
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
110
|
+
d,
|
111
|
+
plugin_name: 'filter',
|
112
|
+
is_default_plugin: false,
|
113
|
+
has_default_config: false,
|
114
|
+
has_ruby_snippet: false
|
115
|
+
)
|
116
|
+
assert_metric_value.call(
|
117
|
+
:enabled_plugins,
|
118
|
+
1,
|
119
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
120
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
121
|
+
d,
|
122
|
+
plugin_name: 'filter/record_transformer',
|
123
|
+
is_default_plugin: false,
|
124
|
+
has_default_config: false,
|
125
|
+
has_ruby_snippet: true
|
126
|
+
)
|
127
|
+
assert_metric_value.call(
|
128
|
+
:enabled_plugins,
|
129
|
+
1,
|
130
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
131
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
132
|
+
d,
|
133
|
+
plugin_name: 'match/stdout',
|
134
|
+
is_default_plugin: false,
|
135
|
+
has_default_config: false,
|
136
|
+
has_ruby_snippet: true
|
137
|
+
)
|
138
|
+
|
139
|
+
# For out_google_cloud, 3 params are present.
|
140
|
+
assert_metric_value.call(
|
141
|
+
:plugin_config,
|
142
|
+
1,
|
143
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
144
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
145
|
+
d,
|
146
|
+
plugin_name: 'google_cloud',
|
147
|
+
param: 'adjust_invalid_timestamps',
|
148
|
+
is_present: true,
|
149
|
+
has_default_config: true
|
150
|
+
)
|
151
|
+
assert_metric_value.call(
|
152
|
+
:plugin_config,
|
153
|
+
1,
|
154
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
155
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
156
|
+
d,
|
157
|
+
plugin_name: 'google_cloud',
|
158
|
+
param: 'autoformat_stackdriver_trace',
|
159
|
+
is_present: true,
|
160
|
+
has_default_config: false
|
161
|
+
)
|
162
|
+
assert_metric_value.call(
|
163
|
+
:plugin_config,
|
164
|
+
1,
|
165
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
166
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
167
|
+
d,
|
168
|
+
plugin_name: 'google_cloud',
|
169
|
+
param: 'coerce_to_utf8',
|
170
|
+
is_present: true,
|
171
|
+
has_default_config: false
|
172
|
+
)
|
173
|
+
# The remaining "google_cloud" params are not present.
|
174
|
+
# The are no params for "detect_exceptions".
|
175
|
+
%w[
|
176
|
+
auth_method
|
177
|
+
detect_json
|
178
|
+
enable_monitoring
|
179
|
+
gcm_service_address
|
180
|
+
grpc_compression_algorithm
|
181
|
+
http_request_key
|
182
|
+
insert_id_key
|
183
|
+
label_map
|
184
|
+
labels
|
185
|
+
labels_key
|
186
|
+
logging_api_url
|
187
|
+
monitoring_type
|
188
|
+
non_utf8_replacement_string
|
189
|
+
operation_key
|
190
|
+
private_key_email
|
191
|
+
private_key_passphrase
|
192
|
+
private_key_path
|
193
|
+
project_id
|
194
|
+
source_location_key
|
195
|
+
span_id_key
|
196
|
+
statusz_port
|
197
|
+
trace_key
|
198
|
+
trace_sampled_key
|
199
|
+
use_grpc
|
200
|
+
use_metadata_service
|
201
|
+
vm_id
|
202
|
+
vm_name
|
203
|
+
zone
|
204
|
+
].each do |p|
|
205
|
+
assert_metric_value.call(
|
206
|
+
:plugin_config,
|
207
|
+
1,
|
208
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
209
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
210
|
+
d,
|
211
|
+
plugin_name: 'google_cloud',
|
212
|
+
param: p,
|
213
|
+
is_present: false,
|
214
|
+
has_default_config: false
|
215
|
+
)
|
216
|
+
end
|
217
|
+
|
218
|
+
# We also export values for the bools.
|
219
|
+
assert_metric_value.call(
|
220
|
+
:config_bool_values,
|
221
|
+
1,
|
222
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
223
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
224
|
+
d,
|
225
|
+
plugin_name: 'google_cloud',
|
226
|
+
param: 'adjust_invalid_timestamps',
|
227
|
+
value: true
|
228
|
+
)
|
229
|
+
assert_metric_value.call(
|
230
|
+
:config_bool_values,
|
231
|
+
1,
|
232
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
233
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
234
|
+
d,
|
235
|
+
plugin_name: 'google_cloud',
|
236
|
+
param: 'autoformat_stackdriver_trace',
|
237
|
+
value: false
|
238
|
+
)
|
239
|
+
assert_metric_value.call(
|
240
|
+
:config_bool_values,
|
241
|
+
1,
|
242
|
+
'agent.googleapis.com/agent/internal/logging/config',
|
243
|
+
OpenCensus::Stats::Aggregation::LastValue,
|
244
|
+
d,
|
245
|
+
plugin_name: 'google_cloud',
|
246
|
+
param: 'coerce_to_utf8',
|
247
|
+
value: true
|
248
|
+
)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
253
|
+
Fluent::Test::FilterTestDriver.new(
|
254
|
+
Fluent::AnalyzeConfigFilter
|
255
|
+
).configure(conf, true)
|
256
|
+
end
|
257
|
+
end
|