fluent-plugin-vadimberezniker-gcp 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. checksums.yaml +7 -0
  2. data/CONTRIBUTING +24 -0
  3. data/Gemfile +3 -0
  4. data/LICENSE +201 -0
  5. data/README.rdoc +53 -0
  6. data/Rakefile +43 -0
  7. data/fluent-plugin-google-cloud.gemspec +43 -0
  8. data/fluent-plugin-vadimberezniker-gcp-0.13.2.gem +0 -0
  9. data/lib/fluent/plugin/common.rb +399 -0
  10. data/lib/fluent/plugin/filter_add_insert_ids.rb +86 -0
  11. data/lib/fluent/plugin/filter_analyze_config.rb +410 -0
  12. data/lib/fluent/plugin/in_object_space_dump.rb +62 -0
  13. data/lib/fluent/plugin/monitoring.rb +265 -0
  14. data/lib/fluent/plugin/out_google_cloud.rb +2209 -0
  15. data/lib/fluent/plugin/statusz.rb +124 -0
  16. data/test/helper.rb +46 -0
  17. data/test/plugin/asserts.rb +87 -0
  18. data/test/plugin/base_test.rb +2680 -0
  19. data/test/plugin/constants.rb +1114 -0
  20. data/test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12 +0 -0
  21. data/test/plugin/data/credentials.json +7 -0
  22. data/test/plugin/data/google-fluentd-baseline.conf +24 -0
  23. data/test/plugin/data/google-fluentd-custom.conf +40 -0
  24. data/test/plugin/data/iam-credentials.json +11 -0
  25. data/test/plugin/data/invalid_credentials.json +8 -0
  26. data/test/plugin/data/new-style-credentials.json +12 -0
  27. data/test/plugin/test_driver.rb +56 -0
  28. data/test/plugin/test_filter_add_insert_ids.rb +137 -0
  29. data/test/plugin/test_filter_analyze_config.rb +257 -0
  30. data/test/plugin/test_out_google_cloud.rb +465 -0
  31. data/test/plugin/test_out_google_cloud_grpc.rb +478 -0
  32. data/test/plugin/utils.rb +148 -0
  33. metadata +347 -0
data/lib/fluent/plugin/statusz.rb ADDED
@@ -0,0 +1,124 @@
1
+ # Copyright 2019 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'erb'
16
+
17
+ # Module for collecting diagnostic information and formatting it as an
18
+ # HTML page to serve on the /statusz endpoint.
19
+ module Statusz
20
+ module_function
21
+
22
+ def response(plugin)
23
+ uptime = Time.now - SERVER_START
24
+ uptime_str = format('%<hours>d hr %<minutes>02d min %<seconds>02d sec',
25
+ hours: uptime / 3600,
26
+ minutes: (uptime / 60) % 60,
27
+ seconds: uptime % 60)
28
+ ERB.new(STATUSZ_TMPL).result_with_hash(
29
+ plugin: plugin,
30
+ uptime_str: uptime_str
31
+ )
32
+ end
33
+ end
34
+
35
+ SERVER_START = Time.now
36
+
37
+ # Does not include the following deprecated config params:
38
+ # auth_method, private_key_email, private_key_passphrase, private_key_path
39
+ CONFIG_KEYS = %w[
40
+ adjust_invalid_timestamps
41
+ autoformat_stackdriver_trace
42
+ coerce_to_utf8
43
+ detect_json
44
+ detect_subservice
45
+ enable_monitoring
46
+ http_request_key
47
+ insert_id_key
48
+ k8s_cluster_location
49
+ k8s_cluster_name
50
+ kubernetes_tag_regexp
51
+ label_map
52
+ labels_key
53
+ labels
54
+ logging_api_url
55
+ monitoring_type
56
+ non_utf8_replacement_string
57
+ operation_key
58
+ project_id
59
+ require_valid_tags
60
+ source_location_key
61
+ span_id_key
62
+ split_logs_by_tag
63
+ statusz_port
64
+ subservice_name
65
+ trace_key
66
+ trace_sampled_key
67
+ use_aws_availability_zone
68
+ use_grpc
69
+ use_metadata_service
70
+ vm_id
71
+ vm_name
72
+ zone
73
+ ].freeze
74
+
75
+ STATUSZ_TMPL = %(\
76
+ <!DOCTYPE html>
77
+ <html>
78
+ <head>
79
+ <title>Status for <%= File.basename($PROGRAM_NAME) %></title>
80
+ <style>
81
+ body {
82
+ font-family: sans-serif;
83
+ }
84
+ h1 {
85
+ clear: both;
86
+ width: 100%;
87
+ text-align: center;
88
+ font-size: 120%;
89
+ background: #eef;
90
+ }
91
+ .lefthand {
92
+ float: left;
93
+ width: 80%;
94
+ }
95
+ .righthand {
96
+ text-align: right;
97
+ }
98
+ td, th {
99
+ background-color: rgba(0, 0, 0, 0.05);
100
+ }
101
+ th {
102
+ text-align: left;
103
+ }
104
+ </style>
105
+ </head>
106
+
107
+ <body>
108
+ <h1>Status for <%= File.basename($PROGRAM_NAME) %></h1>
109
+
110
+ <div>
111
+ <div class="lefthand">
112
+ Started: <%= SERVER_START %><br>
113
+ Up <%= uptime_str %><br>
114
+ </div>
115
+ </div>
116
+
117
+ <h1>Parsed configuration</h1>
118
+
119
+ <table>
120
+ #{CONFIG_KEYS.map { |k| " <tr><th>#{k}</th><td><%= plugin.#{k} %></td></tr>" }.join("\n")}
121
+ </table>
122
+ </body>
123
+ </html>
124
+ ).freeze
data/test/helper.rb ADDED
@@ -0,0 +1,46 @@
1
+ # Copyright 2014 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'rubygems'
16
+ require 'bundler'
17
+ begin
18
+ Bundler.setup(:default, :development)
19
+ rescue Bundler::BundlerError => e
20
+ # rubocop:disable Style/StderrPuts
21
+ $stderr.puts e.message
22
+ $stderr.puts 'Run `bundle install` to install missing gems'
23
+ # rubocop:enable Style/StderrPuts
24
+ exit e.status_code
25
+ end
26
+ require 'test/unit'
27
+
28
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
29
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
30
+ require 'fluent/test'
31
+ unless ENV.key?('VERBOSE')
32
+ nulllogger = Object.new
33
+ nulllogger.instance_eval do |_|
34
+ def respond_to_missing?(_method, _include_private = false)
35
+ true
36
+ end
37
+
38
+ def method_missing(_method, *_args)
39
+ # pass
40
+ end
41
+ end
42
+ # global $log variable is used by fluentd
43
+ $log = nulllogger # rubocop:disable Style/GlobalVars
44
+ end
45
+
46
+ require 'fluent/plugin/out_google_cloud'
data/test/plugin/asserts.rb ADDED
@@ -0,0 +1,87 @@
1
+ # Copyright 2020 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'prometheus/client'
16
+
17
+ # Additional assert functions.
18
+ module Asserts
19
+ # For an optional field with default values, Protobuf omits the field when it
20
+ # is deserialized to json. So we need to add an extra check for gRPC which
21
+ # uses Protobuf.
22
+ #
23
+ # An optional block can be passed in if we need to assert something other than
24
+ # a plain equal. e.g. assert_in_delta.
25
+ def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
26
+ _undefined
27
+ end
28
+
29
+ # Compare the timestamp seconds and nanoseconds with the expected timestamp.
30
+ def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
31
+ assert_equal expected_ts.tv_sec, ts_secs, entry
32
+ # Fluentd v0.14 onwards supports nanosecond timestamp values.
33
+ # Added in 600 ns delta to avoid flaky tests introduced
34
+ # due to rounding error in double-precision floating-point numbers
35
+ # (to account for the missing 9 bits of precision ~ 512 ns).
36
+ # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
37
+ assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
38
+ end
39
+
40
+ # rubocop:disable Metrics/ParameterLists
41
+ def assert_prometheus_metric_value(metric_name, expected_value, _prefix,
42
+ _aggregation, _test_driver, labels = {})
43
+ # rubocop:enable Metrics/ParameterLists
44
+ metric = Prometheus::Client.registry.get(metric_name)
45
+ assert_not_nil(metric)
46
+ metric_value = if labels == :aggregate
47
+ # Sum up all metric values regardless of the labels.
48
+ metric.values.values.reduce(0.0, :+)
49
+ else
50
+ metric.get(labels)
51
+ end
52
+ assert_equal(expected_value, metric_value)
53
+ end
54
+
55
+ # rubocop:disable Metrics/ParameterLists
56
+ def assert_opencensus_metric_value(metric_name, expected_value, prefix,
57
+ aggregation, test_driver, labels = {})
58
+ # rubocop:enable Metrics/ParameterLists
59
+ translator = Monitoring::MetricTranslator.new(metric_name, labels)
60
+ metric_name = translator.name
61
+ labels = translator.translate_labels(labels)
62
+ # The next line collapses the labels to assert against the aggregated data,
63
+ # which can have some labels removed. Without this, it would test against
64
+ # the raw data. The view is more representative of the user experience, even
65
+ # though both tests should work because currently we only aggregate away one
66
+ # label that never changes during runtime.
67
+ labels.select! { |k, _| translator.view_labels.include? k }
68
+ labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
69
+
70
+ registry = test_driver.instance.instance_variable_get(:@registry)
71
+ recorder = registry.instance_variable_get(:@recorders)[prefix]
72
+ view_data = recorder.measure_views_data[metric_name][0].data
73
+ view = recorder.instance_variable_get(:@views)[metric_name]
74
+
75
+ # Assert values in the view.
76
+ assert_kind_of(aggregation, view.aggregation)
77
+ assert_equal(labels.keys, view.columns)
78
+ assert_equal(metric_name, view.measure.name)
79
+ assert_equal('INT64', view.measure.type)
80
+
81
+ # For now assume all metrics are counters.
82
+ tag_values = view.columns.map { |column| labels[column] }
83
+ metric_value = 0
84
+ metric_value = view_data[tag_values].value if view_data.key? tag_values
85
+ assert_equal(expected_value, metric_value)
86
+ end
87
+ end