fluent-plugin-node-exporter-metrics 0.1.0

data/docs/stat.md

@@ -0,0 +1,17 @@
+# Stat Collector
+
+## Metric and label naming
+
+* node_boot_time_seconds
+* node_context_switches_total
+* node_forks_total
+* node_intr_total
+* node_procs_blocked
+* node_procs_running
+
+## Metric and its data sources
+
+Stat collector access the following data sources.
+
+* /proc/stat
+

data/docs/time.md

@@ -0,0 +1,9 @@
+# Time Collector
+
+## Metric and label naming
+
+* node_time_seconds
+
+## Metric and its data sources
+
+*  FluentEventTime.now

data/docs/uname.md

@@ -0,0 +1,15 @@
+# Uname Collector
+
+## Metric and label naming
+
+* node_uname_info {"sysname"=>..., "release"=>..., "version"=>..., "machine"=>..., "nodename"=>..., "domainname"=>...}
+
+
+## Metric and its data sources
+
+Uname collector access the following data sources.
+
+* Etc.uname
+
+NOTE: `Etc.uname` returns at least sysname,release,version,machine,nodename
+but it is not guaranteed to return domainname.

data/docs/vmstat.md

@@ -0,0 +1,17 @@
+# Vmstat Collector
+
+## Metric and label naming
+
+* node_vmstat_oom_kill
+* node_vmstat_pgfault
+* node_vmstat_pgmajfault
+* node_vmstat_pgpgin
+* node_vmstat_pgpgout
+* node_vmstat_pswpin
+* node_vmstat_pswpout
+
+## Metric and its data sources
+
+Vmstat collector access the following data sources.
+
+* /proc/vmstat

data/fluent-plugin-node-exporter-metrics.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-node-exporter-metrics"
+  spec.version = "0.1.0"
+  spec.authors = ["Kentaro Hayashi"]
+  spec.email   = ["hayashi@clear-code.com"]
+
+  spec.summary       = %q{Input plugin which collects metrics similar to Prometheus Node Exporter}
+  spec.description   = %q{node exporter metrics input plugin implements 11 node exporter collectors}
+  spec.homepage      = "https://github.com/fluent-plugins-nursery/fluent-plugin-node-exporter-metrics"
+  spec.license       = "Apache-2.0"
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "cmetrics", "~> 0.2.3"
+  spec.add_runtime_dependency "capng_c", "~> 0.2.2"
+
+  spec.add_development_dependency "bundler", "~> 2.2.0"
+  spec.add_development_dependency "rake", "~> 13.0.6"
+  spec.add_development_dependency "test-unit", "~> 3.4.4"
+  spec.add_development_dependency "test-unit-rr", "~> 1.0.5"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end

data/lib/fluent/plugin/in_node_exporter_metrics.rb

@@ -0,0 +1,138 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "etc"
+require "fluent/env"
+require "fluent/capability"
+require "fluent/plugin/input"
+require "fluent/plugin/node_exporter/cpu_collector"
+require "fluent/plugin/node_exporter/cpufreq_collector"
+require "fluent/plugin/node_exporter/diskstats_collector"
+require "fluent/plugin/node_exporter/filefd_collector"
+require "fluent/plugin/node_exporter/loadavg_collector"
+require "fluent/plugin/node_exporter/meminfo_collector"
+require "fluent/plugin/node_exporter/netdev_collector"
+require "fluent/plugin/node_exporter/stat_collector"
+require "fluent/plugin/node_exporter/time_collector"
+require "fluent/plugin/node_exporter/uname_collector"
+require "fluent/plugin/node_exporter/vmstat_collector"
+
+module Fluent
+  module Plugin
+    class NodeExporterMetricsInput < Fluent::Plugin::Input
+      Fluent::Plugin.register_input("node_exporter_metrics", self)
+
+      helpers :timer
+
+      desc "Interval to scrape metrics from node"
+      config_param :scrape_interval, :time, default: 5
+      desc "Path to mount point to collect process information and metrics"
+      config_param :procfs_path, :string, default: "/proc"
+      desc "Path to file-system to collect system metrics"
+      config_param :sysfs_path, :string, default: "/sys"
+      desc "Tag string"
+      config_param :tag, :string, default: nil
+      desc "Enable cpu collector"
+      config_param :cpu, :bool, default: true
+      desc "Enable cpufreq collector"
+      config_param :cpufreq, :bool, default: true
+      desc "Enable diskstats collector"
+      config_param :diskstats, :bool, default: true
+      desc "Enable filefd collector"
+      config_param :filefd, :bool, default: true
+      desc "Enable loadavg collector"
+      config_param :loadavg, :bool, default: true
+      desc "Enable meminfo collector"
+      config_param :meminfo, :bool, default: true
+      desc "Enable netdev collector"
+      config_param :netdev, :bool, default: true
+      desc "Enable stat collector"
+      config_param :stat, :bool, default: true
+      desc "Enable time collector"
+      config_param :time, :bool, default: true
+      desc "Enable uname collector"
+      config_param :uname, :bool, default: true
+      desc "Enable vmstat collector"
+      config_param :vmstat, :bool, default: true
+
+      def configure(conf)
+        super
+        @collectors = []
+        config = {
+          procfs_path: @procfs_path,
+          sysfs_path: @sysfs_path
+        }
+        @collectors << NodeExporter::CpuMetricsCollector.new(config) if @cpu
+        @collectors << NodeExporter::DiskstatsMetricsCollector.new(config) if @diskstats
+        @collectors << NodeExporter::FilefdMetricsCollector.new(config) if @filefd
+        @collectors << NodeExporter::LoadavgMetricsCollector.new(config) if @loadavg
+        @collectors << NodeExporter::MeminfoMetricsCollector.new(config) if @meminfo
+        @collectors << NodeExporter::NetdevMetricsCollector.new(config) if @netdev
+        @collectors << NodeExporter::StatMetricsCollector.new(config) if @stat
+        @collectors << NodeExporter::TimeMetricsCollector.new(config) if @time
+        @collectors << NodeExporter::UnameMetricsCollector.new(config) if @uname
+        @collectors << NodeExporter::VmstatMetricsCollector.new(config) if @vmstat
+
+        if Fluent.linux?
+          if @cpufreq
+            @capability = Fluent::Capability.new(:current_process)
+            unless Etc.getpwuid.uid != 0 and @capability.have_capability?(:effective, :dac_read_search)
+              raise ConfigError, "Linux capability CAP_DAC_READ_SEARCH must be enabled"
+            end
+            @collectors << NodeExporter::CpufreqMetricsCollector.new(config) if @cpufreq
+          end
+        elsif Fluent.windows?
+          raise ConfigError, "node_exporter_metrics is not supported"
+        end
+
+        if @collectors.empty?
+          raise ConfigError, "all collectors are disabled. Enable at least one collector."
+        end
+      end
+
+      def start
+        super
+        timer_execute(:execute_node_exporter_metrics, @scrape_interval, &method(:refresh_watchers))
+      end
+
+      def refresh_watchers
+        begin
+          @serde = CMetrics::Serde.new
+          @collectors.each do |collector|
+            begin
+              collector.run
+              collector.cmetrics.each do |key, cmetric|
+                @serde.concat(cmetric) if cmetric
+              end
+            rescue => e
+              $log.error(e.message)
+            end
+          end
+          record = {
+            "cmetrics" => @serde.to_msgpack
+          }
+          es = OneEventStream.new(Fluent::EventTime.now, record)
+          router.emit_stream(@tag, es)
+        end
+      rescue => e
+        $log.error(e.message)
+      end
+
+      def shutdown
+      end
+    end
+  end
+end

data/lib/fluent/plugin/node_exporter/cmetrics_dataschema_parser.rb

@@ -0,0 +1,84 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "fluent/plugin/input"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class CMetricsDataSchemaParser
+        def parse(metrics)
+          data = []
+          begin
+            metrics.each do |metric|
+              next if metric["values"].empty?
+              data << to_readable_hash(metric)
+            end
+          rescue => e
+            raise Fluent::ParserError.new(e.message)
+          end
+          data.flatten
+        end
+
+        # Parsed CMetrics Data Schema Format
+        # {
+        #  "name" => metrics name
+        #  "time" => Fluent::EventTime
+        #  "labels" => {"key" => value, ...}
+        #  "value" => ...
+        # }
+        #
+        # "labels" field is optional. It is available when {"meta"=>"label_dictionary"...} and
+        # "labels" in "values" => [{"ts"=>..., "labels"=>[...]}]..
+        #
+        def to_readable_hash(metrics)
+          opts = metrics["meta"]["opts"]
+
+          metric_name = if opts["ss"].size.zero?
+                          "#{opts['ns']}_#{opts['name']}"
+                        else
+                          "#{opts['ns']}_#{opts['ss']}_#{opts['name']}"
+                        end
+          cmetrics = []
+          labels = []
+          unless metrics["meta"]["labels"].empty?
+            metrics["meta"]["labels"].each do |v|
+              labels << metrics["meta"]["label_dictionary"][v]
+            end
+          end
+          metrics["values"].each do |entry|
+            cmetric = {
+              "name" => metric_name,
+              "value" => entry["value"],
+              "desc" => opts["desc"],
+              "ts" => entry["ts"]
+            }
+            unless metrics["meta"]["labels"].empty?
+              params = {}
+              entry["labels"].each_with_index do |v, index|
+                label = labels[index]
+                params[label] = metrics["meta"]["label_dictionary"][v]
+              end
+              cmetric["labels"] = params
+            end
+            cmetrics << cmetric
+          end
+          cmetrics
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/node_exporter/collector.rb

@@ -0,0 +1,41 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "fluent/plugin/input"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class MetricsCollector
+        def initialize(config={})
+          @scrape_interval = config[:scrape_interval] || 5
+          @procfs_path = config[:procfs_path] || "/proc"
+          @sysfs_path = config[:sysfs_path] || "/sys"
+        end
+
+        def scan_sysfs_path(pattern)
+          Dir.glob(File.join(@sysfs_path, pattern)).sort do |a, b|
+            File.basename(a).delete("a-z").to_i <=> File.basename(b).delete("a-z").to_i
+          end
+        end
+
+        def cmetrics
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/node_exporter/cpu_collector.rb

@@ -0,0 +1,130 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "etc"
+require "fluent/plugin/input"
+require "fluent/plugin/node_exporter/collector"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class CpuMetricsCollector < MetricsCollector
+        def initialize(config={})
+          super(config)
+
+          # It varies whether /sys/devices/system/cpu/cpuN/thermal_throttle exists or not
+          @thermal_throttle_path = File.join(@sysfs_path, "devices/system/cpu/cpu0/thermal_throttle")
+
+          @core_throttles_total = if File.exist?(File.join(@thermal_throttle_path, "core_throttle_count"))
+                                    CMetrics::Counter.new
+                                  else
+                                    nil
+                                  end
+          @core_throttles_total.create("node", "cpu", "core_throttles_total",
+                                       "Number of times this CPU core has been throttled.",
+                                       ["core", "package"]) if @core_throttles_total
+
+          @package_throttles_total = if File.exist?(File.join(@thermal_throttle_path, "package_throttle_count"))
+                                       CMetrics::Counter.new
+                                     else
+                                       nil
+                                     end
+          @package_throttles_total.create("node", "cpu", "package_throttles_total",
+                                          "Number of times this CPU package has been throttled.",
+                                          ["package"]) if @package_throttles_total
+
+          @seconds_total = CMetrics::Counter.new
+          @seconds_total.create("node", "cpu", "seconds_total",
+                                "Seconds the CPUs spent in each mode.",
+                                ["cpu", "mode"])
+
+          @guest_seconds_total = CMetrics::Counter.new
+          @guest_seconds_total.create("node", "cpu", "guest_seconds_total",
+                                      "Seconds the CPUs spent in guests (VMs) for each mode.",
+                                      ["cpu", "mode"])
+
+          @core_throttles_set = {}
+          @package_throttles_set = {}
+        end
+
+        def run
+          cpu_thermal_update
+          cpu_stat_update
+        end
+
+        def cpu_thermal_update
+          scan_sysfs_path("devices/system/cpu/cpu[0-9]*").each do |path|
+            next unless @core_throttles_total
+            next unless @package_throttles_total
+            
+            core_id_path = File.join(path, "topology", "core_id")
+            physical_package_path = File.join(path, "topology", "physical_package_id")
+            core_id = File.read(core_id_path).strip
+            physical_package_id = File.read(physical_package_path).strip
+            next if @core_throttles_set[{physical_package_id: physical_package_id, core_id: core_id}]
+            @core_throttles_set[{physical_package_id: physical_package_id, core_id: core_id}] = true
+
+            core_throttle_count = File.read(File.join(path, "thermal_throttle", "core_throttle_count")).to_i
+            @core_throttles_total.set(core_throttle_count, [core_id, physical_package_id])
+
+            next if @package_throttles_set[physical_package_id]
+            @package_throttles_set[physical_package_id] = true
+
+            package_throttle_count = File.read(File.join(path, "thermal_throttle", "package_throttle_count")).to_i
+            @package_throttles_total.set(package_throttle_count, [physical_package_id])
+
+          end
+        end
+
+        STAT_CPU_PATTERN = /^cpu(?<cpuid>\d+)\s(?<user>\d+)\s(?<nice>\d+)\s(?<system>\d+)\s(?<idle>\d+)\s(?<iowait>\d+)\s(?<irq>\d+)\s(?<softirq>\d+)\s(?<steal>\d+)\s(?<guest>\d+)\s(?<guest_nice>\d+)/
+
+        def cpu_stat_update
+          stat_path = File.join(@procfs_path, "stat")
+          File.readlines(stat_path).each do |line|
+            if line.start_with?("cpu ")
+              # Ignore CPU total
+              next
+            elsif line.start_with?("cpu")
+              user_hz = Etc.sysconf(Etc::SC_CLK_TCK)
+              line.match(STAT_CPU_PATTERN) do |m|
+                @seconds_total.set(m[:idle].to_f / user_hz, [m[:cpuid], "idle"])
+                @seconds_total.set(m[:iowait].to_f / user_hz, [m[:cpuid], "iowait"])
+                @seconds_total.set(m[:irq].to_f / user_hz, [m[:cpuid], "irq"])
+                @seconds_total.set(m[:nice].to_f / user_hz, [m[:cpuid], "nice"])
+                @seconds_total.set(m[:softirq].to_f / user_hz, [m[:cpuid], "softirq"])
+                @seconds_total.set(m[:steal].to_f / user_hz, [m[:cpuid], "steal"])
+                @seconds_total.set(m[:system].to_f / user_hz, [m[:cpuid], "system"])
+                @seconds_total.set(m[:user].to_f / user_hz, [m[:cpuid], "user"])
+
+                @guest_seconds_total.set(m[:guest].to_f / user_hz, [m[:cpuid], "user"])
+                @guest_seconds_total.set(m[:guest_nice].to_f / user_hz, [m[:cpuid], "nice"])
+              end
+            end
+          end
+        end
+
+        def cmetrics
+          {
+            core_throttles_total: @core_throttles_total,
+            package_throttles_total: @package_throttles_total,
+            seconds_total: @seconds_total,
+            guest_seconds_total: @guest_seconds_total
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/node_exporter/cpufreq_collector.rb

@@ -0,0 +1,97 @@
+#
+# Copyright 2021- Kentaro Hayashi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "cmetrics"
+require "etc"
+require "fluent/plugin/input"
+require "fluent/plugin/node_exporter/collector"
+
+module Fluent
+  module Plugin
+    module NodeExporter
+      class CpufreqMetricsCollector < MetricsCollector
+        def initialize(config={})
+          super(config)
+
+          if Fluent.linux?
+            @frequency_hertz = CMetrics::Gauge.new
+            @frequency_hertz.create("node", "cpu", "frequency_hertz",
+                                    "Current cpu thread frequency in hertz.", ["cpu"])
+
+            @frequency_max_hertz = CMetrics::Gauge.new
+            @frequency_max_hertz.create("node", "cpu", "frequency_max_hertz",
+                                       "Maximum cpu thread frequency in hertz.", ["cpu"])
+
+            @frequency_min_hertz = CMetrics::Gauge.new
+            @frequency_min_hertz.create("node", "cpu", "frequency_min_hertz",
+                                       "Minimum cpu thread frequency in hertz.", ["cpu"])
+
+            @scaling_frequency_hertz = CMetrics::Gauge.new
+            @scaling_frequency_hertz.create("node", "cpu", "scaling_frequency_hertz",
+                                            "Current scaled CPU thread frequency in hertz.", ["cpu"])
+
+            @scaling_frequency_max_hertz = CMetrics::Gauge.new
+            @scaling_frequency_max_hertz.create("node", "cpu", "scaling_frequency_max_hertz",
+                                       "Maximum scaled CPU thread frequency in hertz.", ["cpu"])
+
+            @scaling_frequency_min_hertz = CMetrics::Gauge.new
+            @scaling_frequency_min_hertz.create("node", "cpu", "scaling_frequency_min_hertz",
+                                       "Minimum scaled CPU thread frequency in hertz.", ["cpu"])
+          end
+        end
+
+        def run
+          cpufreq_update
+        end
+
+        def cpuinfo_cur_freq_exist?
+          path = File.join(@sysfs_path, "devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq")
+          File.exist?(path)
+        end
+
+        def cpufreq_update
+          scan_sysfs_path("devices/system/cpu/cpu[0-9]*").each do |path|
+            next unless Dir.exist?(File.join(path, "cpufreq"))
+
+            cpuinfo_cur_freq_path = File.join(path, "cpufreq", "cpuinfo_cur_freq")
+            cpuinfo_max_freq_path = File.join(path, "cpufreq", "cpuinfo_max_freq")
+            cpuinfo_min_freq_path = File.join(path, "cpufreq", "cpuinfo_min_freq")
+            scaling_cur_freq_path = File.join(path, "cpufreq", "scaling_cur_freq")
+            scaling_max_freq_path = File.join(path, "cpufreq", "scaling_max_freq")
+            scaling_min_freq_path = File.join(path, "cpufreq", "scaling_min_freq")
+            cpu_id = File.basename(path).sub(/cpu(\d+)/, "\\1")
+            @frequency_hertz.set(File.read(cpuinfo_cur_freq_path).to_f, [cpu_id]) if File.exist?(cpuinfo_cur_freq_path)
+            @frequency_max_hertz.set(File.read(cpuinfo_max_freq_path).to_f, [cpu_id]) if File.exist?(cpuinfo_max_freq_path)
+            @frequency_min_hertz.set(File.read(cpuinfo_min_freq_path).to_f, [cpu_id]) if File.exist?(cpuinfo_min_freq_path)
+            @scaling_frequency_hertz.set(File.read(scaling_cur_freq_path).to_f, [cpu_id]) if File.exist?(scaling_cur_freq_path)
+            @scaling_frequency_max_hertz.set(File.read(scaling_max_freq_path).to_f, [cpu_id]) if File.exist?(scaling_max_freq_path)
+            @scaling_frequency_min_hertz.set(File.read(scaling_min_freq_path).to_f, [cpu_id]) if File.exist?(scaling_min_freq_path)
+          end
+        end
+
+        def cmetrics
+          {
+            frequency_hertz: @frequency_hertz,
+            frequency_max_hertz: @frequency_max_hertz,
+            frequency_min_hertz: @frequency_min_hertz,
+            scaling_frequency_hertz: @scaling_frequency_hertz,
+            scaling_frequency_max_hertz: @scaling_frequency_max_hertz,
+            scaling_frequency_min_hertz: @scaling_frequency_min_hertz
+          }
+        end
+      end
+    end
+  end
+end