fluent-plugin-calyptia-monitoring 0.1.0.rc9 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e42e4240470d5d46bcf2d098699d82d1d0eb6440f88fd89ff004d7c79b5789f7
-  data.tar.gz: 7f164695485c5202e003a31defc1b5a4022b67ed25ac71550011eb6400946f43
+  metadata.gz: d30951908ffe3e99416a83ed235545394503652e2052ff85423bbb2216d68c8c
+  data.tar.gz: c0d7fa6e044914592e8dc35bb88c3868e97d0344adf01640c2f517441e32bfc6
 SHA512:
-  metadata.gz: ce39e348d421451fda13460ec7072d35e6e79df6c5bc74dfb48fb8f5d9237ff3c4a97bee8034d6e69ba76540b0c375b0cc599b78606b18adecb146a8b316d24a
-  data.tar.gz: 171bf4f4c6230b68174aafb76d042059956f0404294157c06b6a763ffb4cd2d726fb93cf9dc9956aa598c12ed5cda1aebdba1fac0a8a7af293907d230e91ef18
+  metadata.gz: fe9a55bda096a86b3dd945a42d147da4cd39ace0a11c53ec8effeb70512334104fc2a22c9b853c4fd4a8dd5691a1a0139c7a6a16f2b0029d1bf4b8d98397ecb4
+  data.tar.gz: 7274a5ad7cce4240861e71e480b3706c2b43b782abcfc7c5173ff10a9f3ee22810cba67d31521c54e29c0e414b4cc08f858f1dba11aeb92c939f8663519f3ca0

data/README.md

@@ -40,10 +40,11 @@ $ bundle
 
 |parameter|type|description|default|
 |---|---|---|---|
-|endpoint|string (optional)|The endpoint for Monitoring API HTTP request, e.g. http://example.com/api|`TBD`|
+|endpoint|string (optional)|The endpoint for Monitoring API HTTP request, e.g. http://example.com/api|`"https://cloud-api.calyptia.com"`|
 |api_key|string (required)|The API KEY for Monitoring API HTTP request||
 |rate|time (optional)|Emit monitoring values interval. (minimum interval is 30 seconds.)|`30`|
 |pending_metrics_size|size (optional)|Setup pending metrics capacity size|`100`|
+|fluentd_conf_path|string (optional)|Specify Fluentd config file path for RPC not to be available case|`nil`|
 
 ### Example
 
@@ -84,6 +85,44 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 </source>
 ```
 
+And also retrieving configuration from actual file is also supported:
+
+```aconf
+<system>
+# If users want to use multi workers feature which corresponds to logical number of CPUs, please comment out this line.
+#  workers "#{require 'etc'; Etc.nprocessors}"
+  enable_input_metrics true
+  # This record size measuring settings might impact for performance.
+  # Please be careful for high loaded environment to turn on.
+  enable_size_metrics true
+  <metrics>
+    @type cmetrics
+  </metrics>
+</system>
+# And other configurations....
+
+## Fill YOUR_API_KEY with your Calyptia API KEY
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_moniroting
+  <cloud_monitoring>
+    # endpoint http://development-environment-or-production.fqdn:5000
+    api_key YOUR_API_KEY
+    rate 30
+    pending_metrics_size 100 # Specify capacity for pending metrics
+    fluentd_conf_path /path/to/fluent.conf
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path /path/to/agent/accessible/directories/agent_states
+  </storage>
+</source>
+```
+
+**Note:** We recommend to use RPC version due to some circumstances should differ between a loaded configuration and a saved Fluentd configuration.
+This is because calling dumping config RPC feature can obtain from configuration contents which are loaded on memory. But retrieving configuration from the specified file is just read from the file contents and it cannot handle/retrieve loaded configurations on Fluentd.
+When users just update their Fluentd configurations and forgot to restart/reload their Fluentd instances, loaded configurations differ from just edited ones.
+
 ## Calyptia Monitoring API config generator
 
 Usage:
@@ -104,10 +143,12 @@ Options:
         --disable-get-dump           Disable RPC getDump procedure. getDump is enabled by default.
         --storage-agent-token-dir DIR
                                      Specify accesible storage token dir. (default: /path/to/accesible/dir)
+        --fluentd-conf-path PATH     Specify fluentd configuration file path. (default: nil)
 ```
 
 ## Copyright
 
-* Copyright(c) 2021- Hiroshi Hatake
+* Copyright(c) 2021- Calyptia Inc.
+* Maintainer: Hiroshi Hatake <hatake@calyptia.com>
 * License
   * Apache License, Version 2.0

data/fluent-plugin-calyptia-monitoring.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-calyptia-monitoring"
-  spec.version = "0.1.0.rc9"
+  spec.version = "0.1.0"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["hatake@calyptia.com"]
 
@@ -23,6 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.2.15"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "test-unit", "~> 3.3"
-  spec.add_runtime_dependency "fluentd", [">= 1.14.0.rc", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 1.14.0", "< 2"]
   spec.add_runtime_dependency "fluent-plugin-metrics-cmetrics", ">= 0.1.0.rc3"
 end

data/lib/fluent/command/config_generator.rb

@@ -33,6 +33,8 @@ class CalyptiaConfigGenerator
     @enable_get_dump = true
     @rpc_endpoint = "127.0.0.1:24444"
     @storage_agent_token_dir = default_storage_dir
+    @fluentd_conf_path = nil
+    @disable_rpc = false
 
     prepare_option_parser
   end
@@ -96,6 +98,10 @@ BANNER
     @parser.on("--storage-agent-token-dir DIR", "Specify accesible storage token dir. (default: #{default_storage_dir})") do |s|
       @storage_agent_token_dir = s
     end
+    @parser.on("--fluentd-conf-path PATH", "Specify fluentd configuration file path. (default: nil)") do |s|
+      @fluentd_conf_path = s
+      @disable_rpc = true
+    end
   end
 
   def usage(message = nil)
@@ -125,6 +131,8 @@ BANNER
       size_metrics: @enable_size_metrics,
       enable_get_dump: @enable_get_dump,
       storage_agent_token_dir: @storage_agent_token_dir,
+      fluentd_conf_path: @fluentd_conf_path,
+      disable_rpc: @disable_rpc,
     }
   rescue => e
     usage(e)

data/lib/fluent/plugin/calyptia_config_dumper.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'fluent/env'
+require 'fluent/engine'
+require 'fluent/log'
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/system_config'
+require 'fluent/config/element'
+require 'serverengine'
+require 'stringio'
+
+include Fluent::Configurable
+
+def init_log
+  dl_opts = {}
+  dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+  @sio = StringIO.new('', 'r+')
+  logger = ServerEngine::DaemonLogger.new(@sio, dl_opts)
+  $log = Fluent::Log.new(logger)
+end
+
+init_log
+
+File.open(ENV["FLUENT_CONFIG_PATH"], "r") do |f|
+  config = Fluent::Config.parse(f.read, '(supervisor)', '(readFromFile)', true)
+  system_config = Fluent::SystemConfig.create(config)
+  Fluent::Engine.init(system_config, supervisor_mode: true)
+  Fluent::Engine.run_configure(config, dry_run: true)
+  confs = []
+  masked_element = config.to_masked_element
+  masked_element.elements.each{|e|
+    confs << e.to_s
+  }
+  puts confs.join
+end

data/lib/fluent/plugin/calyptia_monitoring_buffer_ext.rb

@@ -0,0 +1,87 @@
+#
+# fluent-plugin-calyptia-monitoring
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/in_monitor_agent'
+
+module Fluent::Plugin
+  class CalyptiaMonitoringBufferExtInput < MonitorAgentInput
+    CALYPTIA_PLUGIN_BUFFER_METRIC_INFO = {
+      'buffer_total_queued_size' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.total_queued_size_metrics.cmetrics.to_msgpack
+      },
+      'buffer_stage_length' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.stage_length_metrics.cmetrics.to_msgpack
+      },
+      'buffer_stage_byte_size' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.stage_size_metrics.cmetrics.to_msgpack
+      },
+      'buffer_queue_length' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.queue_length_metrics.cmetrics.to_msgpack
+      },
+      'buffer_queue_byte_size' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.queue_size_metrics.cmetrics.to_msgpack
+      },
+      'available_buffer_space_ratios' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.available_buffer_space_ratios_metrics.cmetrics.to_msgpack
+      },
+      'newest_timekey' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.newest_timekey_metrics.cmetrics.to_msgpack
+      },
+      'oldest_timekey' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.oldest_timekey_metrics.cmetrics.to_msgpack
+      },
+    }
+
+    def get_monitor_info(pe, opts = {})
+      obj = {}
+
+      obj['metrics'] = get_plugin_metric(pe)
+
+      obj
+    end
+
+    def get_plugin_metric(pe)
+      # Nop for non output plugin
+      return {} if plugin_category(pe) != "output"
+
+      metrics = {}
+
+      if pe.respond_to?(:statistics)
+        # Force to update buffers' metrics values
+        pe.statistics
+      end
+
+      CALYPTIA_PLUGIN_BUFFER_METRIC_INFO.each_pair { |key, code|
+        begin
+          v = pe.instance_exec(&code)
+          unless v.nil?
+            metrics[key] = v
+          end
+        rescue
+        end
+      }
+
+      metrics
+    end
+  end
+end

data/lib/fluent/plugin/calyptia_monitoring_ext.rb

@@ -27,9 +27,25 @@ module Fluent::Plugin
         @emit_records_metrics.cmetrics.to_msgpack
       },
       'retry_count' => ->(){
-        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        throw(:skip) if @num_errors_metrics.get.nil?
         @num_errors_metrics.cmetrics.to_msgpack
       },
+      'write_count' => ->(){
+        throw(:skip) if @write_count_metrics.get.nil?
+        @write_count_metrics.cmetrics.to_msgpack
+      },
+      'rollback_count' => ->(){
+        throw(:skip) if @rollback_count_metrics.get.nil?
+        @rollback_count_metrics.cmetrics.to_msgpack
+      },
+      'flush_time_count' => ->(){
+        throw(:skip) if @flush_time_count_metrics.get.nil?
+        @flush_time_count_metrics.cmetrics.to_msgpack
+      },
+      'slow_flush_count' => ->(){
+        throw(:skip) if @slow_flush_count_metrics.get.nil?
+        @slow_flush_count_metrics.cmetrics.to_msgpack
+      },
     }
 
     def get_monitor_info(pe, opts = {})

data/lib/fluent/plugin/in_calyptia_monitoring.rb

@@ -17,10 +17,11 @@ require 'net/http'
 require 'monitor'
 require 'time'
 require 'fluent/version'
-require 'fluent/config/element'
-require 'fluent/plugin/metrics'
+require 'fluent/env'
 require "fluent/plugin/input"
+require "serverengine"
 require_relative "calyptia_monitoring_ext"
+require_relative "calyptia_monitoring_buffer_ext"
 require_relative "calyptia_monitoring_calyptia_api_requester"
 
 module Fluent
@@ -31,7 +32,7 @@ module Fluent
       class CreateAgentError < Fluent::ConfigError; end
       class UpdateAgentError < Fluent::ConfigError; end
 
-      helpers :timer, :storage
+      helpers :timer, :storage, :child_process
 
       RPC_CONFIG_DUMP_ENDPOINT = "/api/config.getDump".freeze
       DEFAULT_STORAGE_TYPE = 'local'
@@ -52,6 +53,8 @@ module Fluent
         config_param :rate, :time, default: 30
         desc 'Setup pending metrics capacity size'
         config_param :pending_metrics_size, :size, default: DEFAULT_PENDING_METRICS_SIZE
+        desc 'Specify Fluentd config file path for RPC not to be available case'
+        config_param :fluentd_conf_path, :string, default: nil
       end
 
       def multi_workers_ready?
@@ -67,6 +70,7 @@ module Fluent
 
       def configure(conf)
         super
+
         config = conf.elements.select{|e| e.name == 'storage' }.first
         @storage_agent = storage_create(usage: 'calyptia_monitoring_agent', conf: config, default_type: DEFAULT_STORAGE_TYPE)
       end
@@ -83,6 +87,43 @@ module Fluent
         confs.join
       end
 
+      def get_masked_conf_from_conf_file
+        return "" unless File.exist?(@cloud_monitoring.fluentd_conf_path) # check file existence.
+
+        conf = ""
+        callback = ->(status) {
+          if status && status.success?
+            #nop
+          elsif status
+            log.warn "config dumper exits with error code", prog: prog, status: status.exitstatus, signal: status.termsig
+          else
+            log.warn "config dumper unexpectedly exits without exit status", prog: prog
+          end
+        }
+        spawn_command, arguments = if Fluent.windows?
+                          [::ServerEngine.ruby_bin_path, File.join(File.dirname(__FILE__), "calyptia_config_dumper.rb")]
+                        else
+                          [File.join(File.dirname(__FILE__), "calyptia_config_dumper.rb")]
+                        end
+
+        retval = child_process_execute(:exec_calyptia_config_dumper, spawn_command, arguments: arguments, immediate: true,
+                                       env: {"FLUENT_CONFIG_PATH" => @cloud_monitoring.fluentd_conf_path}, parallel: true, mode: [:read_with_stderr],
+                                       on_exit_callback: callback) do |io|
+          io.set_encoding(Encoding::ASCII_8BIT)
+          conf = io.read
+        end
+        unless retval.nil?
+          begin
+            Timeout.timeout(10) do
+              sleep 0.1 until !conf.empty?
+            end
+          rescue Timeout::Error
+            log.warn "cannot retrive configuration contents on #{@cloud_monitoring.fluentd_conf_path} within 10 seconds."
+          end
+        end
+        conf
+      end
+
       def start
         super
 
@@ -93,13 +134,16 @@ module Fluent
                            end
         raise Fluent::ConfigError, "cmetrics plugin should be used to collect metrics on Calyptia Cloud" unless enabled_cmetrics
         @monitor_agent = Fluent::Plugin::CalyptiaMonitoringExtInput.new
+        @monitor_agent_buffer = Fluent::Plugin::CalyptiaMonitoringBufferExtInput.new
         @api_requester = Fluent::Plugin::CalyptiaAPI::Requester.new(@cloud_monitoring.endpoint,
                                                                     @cloud_monitoring.api_key,
                                                                     log,
                                                                     fluentd_worker_id)
-        if check_config_sending_usability
-          @current_config = get_current_config_from_rpc
-        end
+        @current_config = if !@cloud_monitoring.fluentd_conf_path.nil?
+                            get_masked_conf_from_conf_file
+                          elsif check_config_sending_usability
+                            get_current_config_from_rpc
+                          end
 
         if @cloud_monitoring.rate < 30
           log.warn "High frequency events ingestion is not supported. Set up 30s as ingestion interval"
@@ -222,6 +266,12 @@ module Fluent
             buffer += v
           end
         }
+        @monitor_agent_buffer.plugins_info_all(opts).each {|record|
+          metrics = record["metrics"]
+          metrics.each_pair do |k, v|
+            buffer += v
+          end
+        }
         if buffer.empty?
           log.debug "No initialized metrics is found. Trying to send cmetrics on the next tick."
         else

data/templates/calyptia-conf.erb

@@ -4,8 +4,10 @@
   </metrics>
   enable_input_metrics <%= @enable_input_metrics %>
   enable_size_metrics <%= @enable_size_metrics %>
+<%- unless @disable_rpc -%>
   rpc_endpoint <%= @rpc_endpoint %>
   enable_get_dump <%= @enable_get_dump %>
+<%- end -%>
 <system>
 <source>
   @type calyptia_monitoring
@@ -15,6 +17,9 @@
     endpoint <%= @endpoint %>
 <%- end -%>
     api_key <%= @api_key %>
+<%- if @fluentd_conf_path -%>
+    fluentd_conf_path <%= @fluentd_conf_path %>
+<%- end -%>
   </cloud_monitoring>
   <storage>
     @type local

data/test/command/test_config_generator.rb

@@ -192,6 +192,37 @@ TEXT
     path #{storage_dir}/agent_state
   </storage>
 </source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and fluentd_config_path" do
+      storage_dir = Dir.tmpdir
+      conf_dir = Dir.tmpdir
+      conf_path = File.join(conf_dir, "fluent.conf")
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--storage_agent_token_dir", storage_dir, "--fluentd-conf-path", conf_path]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+    fluentd_conf_path #{conf_path}
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{storage_dir}/agent_state
+  </storage>
+</source>
 TEXT
       assert_equal(expected, dumped_config)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calyptia-monitoring
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc9
+  version: 0.1.0
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-17 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.14.0.rc
+        version: 1.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +68,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.14.0.rc
+        version: 1.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -102,6 +102,8 @@ files:
 - bin/calyptia-config-generator
 - fluent-plugin-calyptia-monitoring.gemspec
 - lib/fluent/command/config_generator.rb
+- lib/fluent/plugin/calyptia_config_dumper.rb
+- lib/fluent/plugin/calyptia_monitoring_buffer_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb
 - lib/fluent/plugin/calyptia_monitoring_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_machine_id.rb
@@ -127,9 +129,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.22
 signing_key: