fluent-plugin-vadimberezniker-gcp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +7 -0
  2. data/CONTRIBUTING +24 -0
  3. data/Gemfile +3 -0
  4. data/LICENSE +201 -0
  5. data/README.rdoc +53 -0
  6. data/Rakefile +43 -0
  7. data/fluent-plugin-google-cloud.gemspec +43 -0
  8. data/fluent-plugin-vadimberezniker-gcp-0.13.2.gem +0 -0
  9. data/lib/fluent/plugin/common.rb +399 -0
  10. data/lib/fluent/plugin/filter_add_insert_ids.rb +86 -0
  11. data/lib/fluent/plugin/filter_analyze_config.rb +410 -0
  12. data/lib/fluent/plugin/in_object_space_dump.rb +62 -0
  13. data/lib/fluent/plugin/monitoring.rb +265 -0
  14. data/lib/fluent/plugin/out_google_cloud.rb +2209 -0
  15. data/lib/fluent/plugin/statusz.rb +124 -0
  16. data/test/helper.rb +46 -0
  17. data/test/plugin/asserts.rb +87 -0
  18. data/test/plugin/base_test.rb +2680 -0
  19. data/test/plugin/constants.rb +1114 -0
  20. data/test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12 +0 -0
  21. data/test/plugin/data/credentials.json +7 -0
  22. data/test/plugin/data/google-fluentd-baseline.conf +24 -0
  23. data/test/plugin/data/google-fluentd-custom.conf +40 -0
  24. data/test/plugin/data/iam-credentials.json +11 -0
  25. data/test/plugin/data/invalid_credentials.json +8 -0
  26. data/test/plugin/data/new-style-credentials.json +12 -0
  27. data/test/plugin/test_driver.rb +56 -0
  28. data/test/plugin/test_filter_add_insert_ids.rb +137 -0
  29. data/test/plugin/test_filter_analyze_config.rb +257 -0
  30. data/test/plugin/test_out_google_cloud.rb +465 -0
  31. data/test/plugin/test_out_google_cloud_grpc.rb +478 -0
  32. data/test/plugin/utils.rb +148 -0
  33. metadata +347 -0
data/lib/fluent/plugin/statusz.rb ADDED
@@ -0,0 +1,124 @@
1
+ # Copyright 2019 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'erb'
16
+
17
+ # Module for collecting diagnostic information and formatting it as an
18
+ # HTML page to serve on the /statusz endpoint.
19
+ module Statusz
20
+ module_function
21
+
22
+ def response(plugin)
23
+ uptime = Time.now - SERVER_START
24
+ uptime_str = format('%<hours>d hr %<minutes>02d min %<seconds>02d sec',
25
+ hours: uptime / 3600,
26
+ minutes: (uptime / 60) % 60,
27
+ seconds: uptime % 60)
28
+ ERB.new(STATUSZ_TMPL).result_with_hash(
29
+ plugin: plugin,
30
+ uptime_str: uptime_str
31
+ )
32
+ end
33
+ end
34
+
35
+ SERVER_START = Time.now
36
+
37
+ # Does not include the following deprecated config params:
38
+ # auth_method, private_key_email, private_key_passphrase, private_key_path
39
+ CONFIG_KEYS = %w[
40
+ adjust_invalid_timestamps
41
+ autoformat_stackdriver_trace
42
+ coerce_to_utf8
43
+ detect_json
44
+ detect_subservice
45
+ enable_monitoring
46
+ http_request_key
47
+ insert_id_key
48
+ k8s_cluster_location
49
+ k8s_cluster_name
50
+ kubernetes_tag_regexp
51
+ label_map
52
+ labels_key
53
+ labels
54
+ logging_api_url
55
+ monitoring_type
56
+ non_utf8_replacement_string
57
+ operation_key
58
+ project_id
59
+ require_valid_tags
60
+ source_location_key
61
+ span_id_key
62
+ split_logs_by_tag
63
+ statusz_port
64
+ subservice_name
65
+ trace_key
66
+ trace_sampled_key
67
+ use_aws_availability_zone
68
+ use_grpc
69
+ use_metadata_service
70
+ vm_id
71
+ vm_name
72
+ zone
73
+ ].freeze
74
+
75
+ STATUSZ_TMPL = %(\
76
+ <!DOCTYPE html>
77
+ <html>
78
+ <head>
79
+ <title>Status for <%= File.basename($PROGRAM_NAME) %></title>
80
+ <style>
81
+ body {
82
+ font-family: sans-serif;
83
+ }
84
+ h1 {
85
+ clear: both;
86
+ width: 100%;
87
+ text-align: center;
88
+ font-size: 120%;
89
+ background: #eef;
90
+ }
91
+ .lefthand {
92
+ float: left;
93
+ width: 80%;
94
+ }
95
+ .righthand {
96
+ text-align: right;
97
+ }
98
+ td, th {
99
+ background-color: rgba(0, 0, 0, 0.05);
100
+ }
101
+ th {
102
+ text-align: left;
103
+ }
104
+ </style>
105
+ </head>
106
+
107
+ <body>
108
+ <h1>Status for <%= File.basename($PROGRAM_NAME) %></h1>
109
+
110
+ <div>
111
+ <div class="lefthand">
112
+ Started: <%= SERVER_START %><br>
113
+ Up <%= uptime_str %><br>
114
+ </div>
115
+ </div>
116
+
117
+ <h1>Parsed configuration</h1>
118
+
119
+ <table>
120
+ #{CONFIG_KEYS.map { |k| " <tr><th>#{k}</th><td><%= plugin.#{k} %></td></tr>" }.join("\n")}
121
+ </table>
122
+ </body>
123
+ </html>
124
+ ).freeze
data/test/helper.rb ADDED
@@ -0,0 +1,46 @@
1
+ # Copyright 2014 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'rubygems'
16
+ require 'bundler'
17
+ begin
18
+ Bundler.setup(:default, :development)
19
+ rescue Bundler::BundlerError => e
20
+ # rubocop:disable Style/StderrPuts
21
+ $stderr.puts e.message
22
+ $stderr.puts 'Run `bundle install` to install missing gems'
23
+ # rubocop:enable Style/StderrPuts
24
+ exit e.status_code
25
+ end
26
+ require 'test/unit'
27
+
28
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
29
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
30
+ require 'fluent/test'
31
+ unless ENV.key?('VERBOSE')
32
+ nulllogger = Object.new
33
+ nulllogger.instance_eval do |_|
34
+ def respond_to_missing?(_method, _include_private = false)
35
+ true
36
+ end
37
+
38
+ def method_missing(_method, *_args)
39
+ # pass
40
+ end
41
+ end
42
+ # global $log variable is used by fluentd
43
+ $log = nulllogger # rubocop:disable Style/GlobalVars
44
+ end
45
+
46
+ require 'fluent/plugin/out_google_cloud'
data/test/plugin/asserts.rb ADDED
@@ -0,0 +1,87 @@
1
+ # Copyright 2020 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'prometheus/client'
16
+
17
+ # Additional assert functions.
18
+ module Asserts
19
+ # For an optional field with default values, Protobuf omits the field when it
20
+ # is deserialized to json. So we need to add an extra check for gRPC which
21
+ # uses Protobuf.
22
+ #
23
+ # An optional block can be passed in if we need to assert something other than
24
+ # a plain equal. e.g. assert_in_delta.
25
+ def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
26
+ _undefined
27
+ end
28
+
29
+ # Compare the timestamp seconds and nanoseconds with the expected timestamp.
30
+ def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
31
+ assert_equal expected_ts.tv_sec, ts_secs, entry
32
+ # Fluentd v0.14 onwards supports nanosecond timestamp values.
33
+ # Added in 600 ns delta to avoid flaky tests introduced
34
+ # due to rounding error in double-precision floating-point numbers
35
+ # (to account for the missing 9 bits of precision ~ 512 ns).
36
+ # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
37
+ assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
38
+ end
39
+
40
+ # rubocop:disable Metrics/ParameterLists
41
+ def assert_prometheus_metric_value(metric_name, expected_value, _prefix,
42
+ _aggregation, _test_driver, labels = {})
43
+ # rubocop:enable Metrics/ParameterLists
44
+ metric = Prometheus::Client.registry.get(metric_name)
45
+ assert_not_nil(metric)
46
+ metric_value = if labels == :aggregate
47
+ # Sum up all metric values regardless of the labels.
48
+ metric.values.values.reduce(0.0, :+)
49
+ else
50
+ metric.get(labels)
51
+ end
52
+ assert_equal(expected_value, metric_value)
53
+ end
54
+
55
+ # rubocop:disable Metrics/ParameterLists
56
+ def assert_opencensus_metric_value(metric_name, expected_value, prefix,
57
+ aggregation, test_driver, labels = {})
58
+ # rubocop:enable Metrics/ParameterLists
59
+ translator = Monitoring::MetricTranslator.new(metric_name, labels)
60
+ metric_name = translator.name
61
+ labels = translator.translate_labels(labels)
62
+ # The next line collapses the labels to assert against the aggregated data,
63
+ # which can have some labels removed. Without this, it would test against
64
+ # the raw data. The view is more representative of the user experience, even
65
+ # though both tests should work because currently we only aggregate away one
66
+ # label that never changes during runtime.
67
+ labels.select! { |k, _| translator.view_labels.include? k }
68
+ labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
69
+
70
+ registry = test_driver.instance.instance_variable_get(:@registry)
71
+ recorder = registry.instance_variable_get(:@recorders)[prefix]
72
+ view_data = recorder.measure_views_data[metric_name][0].data
73
+ view = recorder.instance_variable_get(:@views)[metric_name]
74
+
75
+ # Assert values in the view.
76
+ assert_kind_of(aggregation, view.aggregation)
77
+ assert_equal(labels.keys, view.columns)
78
+ assert_equal(metric_name, view.measure.name)
79
+ assert_equal('INT64', view.measure.type)
80
+
81
+ # For now assume all metrics are counters.
82
+ tag_values = view.columns.map { |column| labels[column] }
83
+ metric_value = 0
84
+ metric_value = view_data[tag_values].value if view_data.key? tag_values
85
+ assert_equal(expected_value, metric_value)
86
+ end
87
+ end