fluent-plugin-node-exporter-metrics 0.1.0

data/lib/fluent/plugin/node_exporter/time_collector.rb

@@ -0,0 +1,50 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "fluent/plugin/input"
+require "fluent/plugin/node_exporter/collector"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class TimeMetricsCollector < MetricsCollector
+        def initialize(config={})
+          super(config)
+
+          @gauge = CMetrics::Gauge.new
+          @gauge.create("node", "", "time_seconds",
+                        "System time in seconds since epoch (1970).")
+        end
+
+        def run
+          time_update
+        end
+
+        def time_update
+          current_time = Fluent::EventTime.now
+          value = current_time.to_i / 1e9
+          @gauge.set(value)
+        end
+
+        def cmetrics
+          {
+            time_seconds: @gauge
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/node_exporter/uname_collector.rb

@@ -0,0 +1,64 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "etc"
+require "fluent/plugin/input"
+require "fluent/plugin/node_exporter/collector"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class UnameMetricsCollector < MetricsCollector
+        def initialize(config={})
+          super(config)
+
+          @gauge = CMetrics::Gauge.new
+          @gauge.create("node", "uname", "info",
+                        "Labeled system information as provided by the uname system call.",
+                        ["sysname", "release", "version", "machine", "nodename", "domainname"])
+        end
+
+        def run
+          uname_update
+        end
+
+        def uname_update
+          # Etc.uname returns at least sysname,release,version,machine,nodename
+          # but it is not guaranteed to return domainname.
+          domainname = if Etc.uname.has_key?(:domainname)
+                         Etc.uname[:domainname]
+                       else
+                         "(none)"
+                       end
+          # Use 1 explicitly for default gauge value
+          @gauge.set(1, [
+                       Etc.uname[:sysname],
+                       Etc.uname[:release],
+                       Etc.uname[:version],
+                       Etc.uname[:machine],
+                       Etc.uname[:nodename],
+                       domainname])
+        end
+
+        def cmetrics
+          {
+            info: @gauge
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/node_exporter/vmstat_collector.rb

@@ -0,0 +1,56 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "fluent/plugin/input"
+require "fluent/plugin/node_exporter/collector"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class VmstatMetricsCollector < MetricsCollector
+
+        VMSTAT_ENTRIES_REGEX = /^(oom_kill|pgpg|pswp|pg.*fault).*/
+
+        def initialize(config={})
+          super(config)
+
+          @metrics = {}
+        end
+
+        def run
+          vmstat_update
+        end
+
+        def vmstat_update
+          vmstat_path = File.join(@procfs_path, "vmstat")
+          File.readlines(vmstat_path).each do |line|
+            if VMSTAT_ENTRIES_REGEX.match?(line)
+              key, value = line.split(' ', 2)
+              @untyped = CMetrics::Untyped.new
+              @untyped.create("node", "vmstat", key, "#{vmstat_path} information field #{key}.")
+              @untyped.set(value.to_f)
+              @metrics[key.intern] = @untyped
+            end
+          end
+        end
+
+        def cmetrics
+          @metrics
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_node_exporter_metrics.rb

@@ -0,0 +1,54 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/parser"
+require "fluent/msgpack_factory"
+
+require "fluent/plugin/node_exporter/cmetrics_dataschema_parser"
+
+module Fluent
+  module Plugin
+    class NodeExporterMetricsParser < Parser
+      Plugin.register_parser('node_exporter_metrics', self)
+
+      def configure(conf)
+        super
+        @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
+        @parser = Fluent::Plugin::NodeExporter::CMetricsDataSchemaParser.new
+      end
+
+      def parser_type
+        :binary
+      end
+
+      def parse(data)
+        @unpacker.feed_each(data) do |obj|
+          metrics = @parser.parse(obj)
+          yield Fluent::EventTime.now, metrics
+        end
+      end
+
+      alias parse_partial_data parse
+
+      def parse_io(io, &block)
+        u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
+        u.each do |obj|
+          metrics = @parser.parse(obj)
+          yield Fluent::EventTime.now, metrics
+        end
+      end
+    end
+  end
+end

data/test/fixtures/cpu/with_thermal_throttle/proc/stat

@@ -0,0 +1,3 @@
+cpu  464630 486 142953 41892706 67964 0 33936 0 0 0
+cpu0 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000
+cpu1 30454 2400 7030 2620182 3768 0 5700 0 0 0

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count

@@ -0,0 +1 @@
+1

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu0/thermal_throttle/package_throttle_count

@@ -0,0 +1 @@
+10

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu0/topology/core_id

@@ -0,0 +1 @@
+0

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu0/topology/physical_package_id

@@ -0,0 +1 @@
+0

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu1/thermal_throttle/core_throttle_count

@@ -0,0 +1 @@
+2

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu1/thermal_throttle/package_throttle_count

@@ -0,0 +1 @@
+1

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu1/topology/core_id

@@ -0,0 +1 @@
+1

data/test/fixtures/cpu/with_thermal_throttle/sys/devices/system/cpu/cpu1/topology/physical_package_id

@@ -0,0 +1 @@
+0

data/test/fixtures/cpu/without_thermal_throttle/proc/stat

@@ -0,0 +1,3 @@
+cpu  464630 486 142953 41892706 67964 0 33936 0 0 0
+cpu0 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000
+cpu1 30454 2400 7030 2620182 3768 0 5700 0 0 0

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq

@@ -0,0 +1 @@
+2200000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq

@@ -0,0 +1 @@
+2500000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq

@@ -0,0 +1 @@
+2000000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq

@@ -0,0 +1 @@
+2600000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq

@@ -0,0 +1 @@
+2900000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq

@@ -0,0 +1 @@
+2400000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_cur_freq

@@ -0,0 +1 @@
+2300000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_max_freq

@@ -0,0 +1 @@
+2400000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_min_freq

@@ -0,0 +1 @@
+2100000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/scaling_cur_freq

@@ -0,0 +1 @@
+2900000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/scaling_max_freq

@@ -0,0 +1 @@
+3000000

data/test/fixtures/cpufreq/with_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/scaling_min_freq

@@ -0,0 +1 @@
+2700000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq

@@ -0,0 +1 @@
+2500000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq

@@ -0,0 +1 @@
+2000000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq

@@ -0,0 +1 @@
+2600000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq

@@ -0,0 +1 @@
+2900000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq

@@ -0,0 +1 @@
+2400000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_max_freq

@@ -0,0 +1 @@
+2400000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/cpuinfo_min_freq

@@ -0,0 +1 @@
+2100000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/scaling_cur_freq

@@ -0,0 +1 @@
+2900000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/scaling_max_freq

@@ -0,0 +1 @@
+3000000

data/test/fixtures/cpufreq/without_cur_freq/sys/devices/system/cpu/cpu1/cpufreq/scaling_min_freq

@@ -0,0 +1 @@
+2700000

data/test/helper.rb

@@ -0,0 +1,24 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "test/unit/rr"
+require "fluent/test"
+require "fluent/test/driver/input"
+require "fluent/test/helpers"
+require "fluent/plugin/in_node_exporter_metrics"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)
+
+def fixture_filesystem_root(collector, test_case, filesystem)
+  File.expand_path(File.join(File.dirname(__FILE__), "fixtures",
+                             collector, test_case, filesystem))
+end
+
+def fixture_sysfs_root(collector, test_case="")
+  fixture_filesystem_root(collector, test_case, "sys")
+end
+
+def fixture_procfs_root(collector, test_case="")
+  fixture_filesystem_root(collector, test_case, "proc")
+end
+

data/test/plugin/test_cmetrics_data_schema_parser.rb

@@ -0,0 +1,83 @@
+require "helper"
+require "fluent/plugin/node_exporter/cmetrics_dataschema_parser"
+
+class CMetricsDataSchemaParserTest < Test::Unit::TestCase
+
+  NSEC_IN_SECONDS = 1000 * 1000 * 1000
+
+  setup do
+    @parser = Fluent::Plugin::NodeExporter::CMetricsDataSchemaParser.new
+    @nsecs = Time.parse("2021/09/01").to_i * NSEC_IN_SECONDS
+  end
+
+  sub_test_case "parser" do
+    def test_without_labels
+      objs = [{"meta"=>
+               {"ver"=>2,
+                "type"=>1,
+                "opts"=>
+                {"ns"=>"node",
+                 "ss"=>"",
+                 "name"=>"time_seconds",
+                 "desc"=>"System time in seconds since epoch (1970)."},
+                "label_dictionary"=>[],
+                "static_labels"=>[],
+                "labels"=>[]},
+               "values"=>[{"ts"=>@nsecs, "value"=>1.632131027}]}]
+      assert_equal([
+                     {
+                       "desc"=>"System time in seconds since epoch (1970).",
+                       "name"=>"node_time_seconds",
+                       "value"=>1.632131027,
+                       "ts"=>@nsecs
+                     }
+                   ],
+                   @parser.parse(objs))
+    end
+
+    def test_with_labels
+      objs = [{"meta"=>
+               {"ver"=>2,
+                "type"=>1,
+                "opts"=>
+                {"ns"=>"node",
+                 "ss"=>"uname",
+                 "name"=>"info",
+                 "desc"=>
+                 "Labeled system information as provided by the uname system call."},
+                "label_dictionary"=>
+                ["sysname",
+                 "release",
+                 "version",
+                 "machine",
+                 "nodename",
+                 "domainname",
+                 "Linux",
+                 "5.10.0-8-amd64",
+                 "#1 SMP Debian",
+                 "x86_64",
+                 "jet",
+                 "(none)"],
+                "static_labels"=>[],
+                "labels"=>[0, 1, 2, 3, 4, 5]},
+               "values"=>
+               [{"ts"=>@nsecs,
+                 "value"=>1.0,
+                 "labels"=>[6, 7, 8, 9, 10, 11]}]}]
+      assert_equal([
+                     {"desc"=>"Labeled system information as provided by the uname system call.",
+                      "labels"=>
+                      {"domainname"=>"(none)",
+                       "machine"=>"x86_64",
+                       "nodename"=>"jet",
+                       "release"=>"5.10.0-8-amd64",
+                       "sysname"=>"Linux",
+                       "version"=>"#1 SMP Debian"},
+                      "name"=>"node_uname_info",
+                      "ts"=>@nsecs,
+                      "value"=>1.0}
+                   ],
+                   @parser.parse(objs))
+    end
+  end
+end

data/test/plugin/test_cpu_collector.rb

@@ -0,0 +1,84 @@
+require "helper"
+require "fluent/plugin/in_node_exporter_metrics"
+require "fluent/plugin/node_exporter/cpu_collector"
+
+class CpuColectorTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  teardown do
+    Fluent::Engine.stop
+  end
+
+  CONFIG = config_element("ROOT", "", {
+                            "scrape_interval" => 5,
+                            "procfs_path" => "/proc",
+                            "sysfs_path" => "/sys"
+                          })
+
+  DEFAULT_COLLECTORS = CONFIG + config_element("", "", {
+                                               })
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::NodeExporterMetricsInput).configure(conf)
+  end
+
+  sub_test_case "cpu_thermal_throttle" do
+    def test_cpu_thermal_throttle
+      config = {
+        scpape_interval: 1,
+        procfs_path: fixture_procfs_root("cpu", "with_thermal_throttle"),
+        sysfs_path: fixture_sysfs_root("cpu", "with_thermal_throttle")
+      }
+      collector = Fluent::Plugin::NodeExporter::CpuMetricsCollector.new(config)
+      collector.run
+      # CPU0/1 thermal throttle
+      core_throttles_total = collector.cmetrics[:core_throttles_total]
+      package_throttles_total = collector.cmetrics[:package_throttles_total]
+      assert_equal([1.0, 2.0, 10.0],
+                   [core_throttles_total.val(["0", "0"]),
+                    core_throttles_total.val(["1", "0"]),
+                    package_throttles_total.val(["0"])
+                   ])
+    end
+
+    def test_without_cpu_thermal_throttle
+      config = {
+        scpape_interval: 1,
+        procfs_path: fixture_procfs_root("cpu", "without_thermal_throttle"),
+        sysfs_path: fixture_sysfs_root("cpu", "without_thermal_throttle")
+      }
+      collector = Fluent::Plugin::NodeExporter::CpuMetricsCollector.new(config)
+      collector.run
+      assert_equal([nil, nil],
+                   [collector.cmetrics[:core_throttles_total],
+                    collector.cmetrics[:package_throttles_total]])
+    end
+
+    def test_cpu_stat
+      config = {
+        scpape_interval: 1,
+        procfs_path: fixture_procfs_root("cpu", "with_thermal_throttle"),
+        sysfs_path: fixture_sysfs_root("cpu", "with_thermal_throttle")
+      }
+      collector = Fluent::Plugin::NodeExporter::CpuMetricsCollector.new(config)
+      stub(Etc).sysconf { 1000 }
+      collector.run
+      seconds_total = collector.cmetrics[:seconds_total]
+      guest_seconds_total = collector.cmetrics[:guest_seconds_total]
+      assert_equal([4.0, 5.0, 6.0, 2.0, 7.0, 8.0, 3.0, 1.0, 9.0, 10.0],
+                   [seconds_total.val(["0", "idle"]),
+                    seconds_total.val(["0", "iowait"]),
+                    seconds_total.val(["0", "irq"]),
+                    seconds_total.val(["0", "nice"]),
+                    seconds_total.val(["0", "softirq"]),
+                    seconds_total.val(["0", "steal"]),
+                    seconds_total.val(["0", "system"]),
+                    seconds_total.val(["0", "user"]),
+                    guest_seconds_total.val(["0", "user"]),
+                    guest_seconds_total.val(["0", "nice"]),
+                   ])
+    end
+  end
+end

data/test/plugin/test_cpufreq_collector.rb

@@ -0,0 +1,75 @@
+require "helper"
+require "fluent/plugin/in_node_exporter_metrics"
+require "fluent/plugin/node_exporter/cpufreq_collector"
+
+class CpufreqColectorTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @capability = Fluent::Capability.new
+  end
+
+  teardown do
+    Fluent::Engine.stop
+  end
+
+  CONFIG = config_element("ROOT", "", {
+                            "scrape_interval" => 5,
+                            "procfs_path" => "/proc",
+                            "sysfs_path" => "/sys"
+                          })
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::NodeExporterMetricsInput).configure(conf)
+  end
+
+  def cpufreq_available?
+    freq_path = "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq"
+    Dir.exist?("/sys/devices/system/cpu/cpu0/cpufreq") and
+      File.exist?(freq_path) and
+      (File.readable?(freq_path) or @capability.have_capability?(:effective, :dac_read_search))
+  end
+
+  sub_test_case "cpufreq" do
+    data(
+      with: ["with_cur_freq", [
+               2200000.0, 2500000.0, 2000000.0,
+               2300000.0, 2400000.0, 2100000.0,
+               2600000.0, 2900000.0, 2400000.0,
+               2900000.0, 3000000.0, 2700000.0]],
+      without: ["without_cur_freq", [
+                  nil, 2500000.0, 2000000.0,
+                  nil, 2400000.0, 2100000.0,
+                  2600000.0, 2900000.0, 2400000.0,
+                  2900000.0, 3000000.0, 2700000.0]]
+    )
+    test "cpuinfo_cur_frequency" do |(fixture, expected)|
+      omit "skip assertion when cpufreq is not available" unless cpufreq_available?
+
+      config = {
+        sysfs_path: fixture_sysfs_root("cpufreq", fixture)
+      }
+      collector = Fluent::Plugin::NodeExporter::CpufreqMetricsCollector.new(config)
+      collector.run
+      frequency_hertz = collector.cmetrics[:frequency_hertz]
+      frequency_max_hertz = collector.cmetrics[:frequency_max_hertz]
+      frequency_min_hertz = collector.cmetrics[:frequency_min_hertz]
+      scaling_frequency_hertz = collector.cmetrics[:scaling_frequency_hertz]
+      scaling_frequency_max_hertz = collector.cmetrics[:scaling_frequency_max_hertz]
+      scaling_frequency_min_hertz = collector.cmetrics[:scaling_frequency_min_hertz]
+      assert_equal(expected,
+                   [frequency_hertz.val(["0"]),
+                    frequency_max_hertz.val(["0"]),
+                    frequency_min_hertz.val(["0"]),
+                    frequency_hertz.val(["1"]),
+                    frequency_max_hertz.val(["1"]),
+                    frequency_min_hertz.val(["1"]),
+                    scaling_frequency_hertz.val(["0"]),
+                    scaling_frequency_max_hertz.val(["0"]),
+                    scaling_frequency_min_hertz.val(["0"]),
+                    scaling_frequency_hertz.val(["1"]),
+                    scaling_frequency_max_hertz.val(["1"]),
+                    scaling_frequency_min_hertz.val(["1"]),
+                   ])
+    end
+  end
+end