fluent-plugin-calyptia-monitoring 0.1.0.rc6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc96c5b09c542854a7fc86e735d01c1b8d15d6ffcacc35d21bc4e02578f7d379
-  data.tar.gz: 1deccf93ccb4b41f802099d72b33670459d78c65917af979dd074041de6b964f
+  metadata.gz: d30951908ffe3e99416a83ed235545394503652e2052ff85423bbb2216d68c8c
+  data.tar.gz: c0d7fa6e044914592e8dc35bb88c3868e97d0344adf01640c2f517441e32bfc6
 SHA512:
-  metadata.gz: ba2e21ff1a791fcf81d33d3c568874ac672c3b9882e942007e57c403f2f0d0a12d211b473244bff8303f58bb5b853f2d9acb6c2d527e0d9623484c0a626f26b1
-  data.tar.gz: 62b0c374c5917ca03d8b836fdf2508838e4cc134d04e25873021fd972fa9914f5a38888e2de62d335b5576883307c343ec72b3f00d5959eb502cf9cd8ec9e87d
+  metadata.gz: fe9a55bda096a86b3dd945a42d147da4cd39ace0a11c53ec8effeb70512334104fc2a22c9b853c4fd4a8dd5691a1a0139c7a6a16f2b0029d1bf4b8d98397ecb4
+  data.tar.gz: 7274a5ad7cce4240861e71e480b3706c2b43b782abcfc7c5173ff10a9f3ee22810cba67d31521c54e29c0e414b4cc08f858f1dba11aeb92c939f8663519f3ca0

data/README.md

@@ -40,10 +40,11 @@ $ bundle
 
 |parameter|type|description|default|
 |---|---|---|---|
-|endpoint|string (optional)|The endpoint for Monitoring API HTTP request, e.g. http://example.com/api|`TBD`|
+|endpoint|string (optional)|The endpoint for Monitoring API HTTP request, e.g. http://example.com/api|`"https://cloud-api.calyptia.com"`|
 |api_key|string (required)|The API KEY for Monitoring API HTTP request||
 |rate|time (optional)|Emit monitoring values interval. (minimum interval is 30 seconds.)|`30`|
 |pending_metrics_size|size (optional)|Setup pending metrics capacity size|`100`|
+|fluentd_conf_path|string (optional)|Specify Fluentd config file path for RPC not to be available case|`nil`|
 
 ### Example
 
@@ -84,8 +85,70 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 </source>
 ```
 
+And also retrieving configuration from actual file is also supported:
+
+```aconf
+<system>
+# If users want to use multi workers feature which corresponds to logical number of CPUs, please comment out this line.
+#  workers "#{require 'etc'; Etc.nprocessors}"
+  enable_input_metrics true
+  # This record size measuring settings might impact for performance.
+  # Please be careful for high loaded environment to turn on.
+  enable_size_metrics true
+  <metrics>
+    @type cmetrics
+  </metrics>
+</system>
+# And other configurations....
+
+## Fill YOUR_API_KEY with your Calyptia API KEY
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_moniroting
+  <cloud_monitoring>
+    # endpoint http://development-environment-or-production.fqdn:5000
+    api_key YOUR_API_KEY
+    rate 30
+    pending_metrics_size 100 # Specify capacity for pending metrics
+    fluentd_conf_path /path/to/fluent.conf
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path /path/to/agent/accessible/directories/agent_states
+  </storage>
+</source>
+```
+
+**Note:** We recommend to use RPC version due to some circumstances should differ between a loaded configuration and a saved Fluentd configuration.
+This is because calling dumping config RPC feature can obtain from configuration contents which are loaded on memory. But retrieving configuration from the specified file is just read from the file contents and it cannot handle/retrieve loaded configurations on Fluentd.
+When users just update their Fluentd configurations and forgot to restart/reload their Fluentd instances, loaded configurations differ from just edited ones.
+
+## Calyptia Monitoring API config generator
+
+Usage:
+
+```
+Usage: calyptia-config-generator api_key [options]
+
+Generate Calyptia monitoring plugin config definitions
+
+Arguments:
+	api_key: Specify your API_KEY
+
+Options:
+        --endpoint URL               API Endpoint URL (default: nil, and if omitted, using default endpoint)
+        --rpc-endpoint URL           Specify RPC Endpoint URL (default: 127.0.0.1:24444)
+        --disable-input-metrics      Disable Input plugin metrics. Input metrics is enabled by default
+        --enable-size-metrics        Enable event size metrics. Size metrics is disabled by default.
+        --disable-get-dump           Disable RPC getDump procedure. getDump is enabled by default.
+        --storage-agent-token-dir DIR
+                                     Specify accesible storage token dir. (default: /path/to/accesible/dir)
+        --fluentd-conf-path PATH     Specify fluentd configuration file path. (default: nil)
+```
+
 ## Copyright
 
-* Copyright(c) 2021- Hiroshi Hatake
+* Copyright(c) 2021- Calyptia Inc.
+* Maintainer: Hiroshi Hatake <hatake@calyptia.com>
 * License
   * Apache License, Version 2.0

data/bin/calyptia-config-generator

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+require 'fluent/command/config_generator'
+
+CalyptiaConfigGenerator.new.call

data/fluent-plugin-calyptia-monitoring.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-calyptia-monitoring"
-  spec.version = "0.1.0.rc6"
+  spec.version = "0.1.0"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["hatake@calyptia.com"]
 
@@ -23,6 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.2.15"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "test-unit", "~> 3.3"
-  spec.add_runtime_dependency "fluentd", [">= 1.13.0", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 1.14.0", "< 2"]
   spec.add_runtime_dependency "fluent-plugin-metrics-cmetrics", ">= 0.1.0.rc3"
 end

data/lib/fluent/command/config_generator.rb

@@ -0,0 +1,144 @@
+#
+# fluent-plugin-calyptia-monitoring
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+require 'fluent/version'
+
+class CalyptiaConfigGenerator
+  def initialize(argv = ARGV)
+    @argv = argv
+    @api_key = nil
+    @endpoint = nil
+    @enable_input_metrics = true
+    @enable_size_metrics = false
+    @enable_get_dump = true
+    @rpc_endpoint = "127.0.0.1:24444"
+    @storage_agent_token_dir = default_storage_dir
+    @fluentd_conf_path = nil
+    @disable_rpc = false
+
+    prepare_option_parser
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  def call
+    parse_options!
+
+    puts dump_configuration_for_calyptia
+  end
+
+  def dump_configuration_for_calyptia
+    dumped = ""
+    template = template_path("calyptia-conf.erb").read
+
+    dumped <<
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(template, trim_mode: "-")
+      else
+        ERB.new(template, nil, "-")
+      end.result(binding)
+    dumped
+  end
+  private
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: #{$0} api_key [options]
+
+Generate Calyptia monitoring plugin config definitions
+
+Arguments:
+\tapi_key: Specify your API_KEY
+
+Options:
+BANNER
+    @parser.on("--endpoint URL", "API Endpoint URL (default: nil, and if omitted, using default endpoint)") do |s|
+      @endpoint = s
+    end
+    @parser.on("--rpc-endpoint URL", "Specify RPC Endpoint URL (default: 127.0.0.1:24444)") do |s|
+      @rpc_endpoint = s
+    end
+    @parser.on("--disable-input-metrics", "Disable Input plugin metrics. Input metrics is enabled by default") do
+      @enable_input_metrics = false
+    end
+    @parser.on("--enable-size-metrics", "Enable event size metrics. Size metrics is disabled by default.") do
+      @enable_size_metrics = true
+    end
+    @parser.on("--disable-get-dump", "Disable RPC getDump procedure. getDump is enabled by default.") do
+      @enable_get_dump = false
+    end
+    @parser.on("--storage-agent-token-dir DIR", "Specify accesible storage token dir. (default: #{default_storage_dir})") do |s|
+      @storage_agent_token_dir = s
+    end
+    @parser.on("--fluentd-conf-path PATH", "Specify fluentd configuration file path. (default: nil)") do |s|
+      @fluentd_conf_path = s
+      @disable_rpc = true
+    end
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify api_key" unless @argv.size == 1
+
+    host, port = @rpc_endpoint.split(':')
+    if @enable_get_dump && (!host || !port)
+      puts "Error: invalid rpc_endpoint format. Specify `host:port' form."
+      exit(false)
+    end
+
+    @api_key, = @argv
+    @options = {
+      api_key: @api_key,
+      endpoint: @endpoint,
+      rpc_endpoint: @rpc_endpoint,
+      input_metrics: @enable_input_metrics,
+      size_metrics: @enable_size_metrics,
+      enable_get_dump: @enable_get_dump,
+      storage_agent_token_dir: @storage_agent_token_dir,
+      fluentd_conf_path: @fluentd_conf_path,
+      disable_rpc: @disable_rpc,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/#{name}").realpath
+  end
+end

data/lib/fluent/plugin/calyptia_config_dumper.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'fluent/env'
+require 'fluent/engine'
+require 'fluent/log'
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/system_config'
+require 'fluent/config/element'
+require 'serverengine'
+require 'stringio'
+
+include Fluent::Configurable
+
+def init_log
+  dl_opts = {}
+  dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+  @sio = StringIO.new('', 'r+')
+  logger = ServerEngine::DaemonLogger.new(@sio, dl_opts)
+  $log = Fluent::Log.new(logger)
+end
+
+init_log
+
+File.open(ENV["FLUENT_CONFIG_PATH"], "r") do |f|
+  config = Fluent::Config.parse(f.read, '(supervisor)', '(readFromFile)', true)
+  system_config = Fluent::SystemConfig.create(config)
+  Fluent::Engine.init(system_config, supervisor_mode: true)
+  Fluent::Engine.run_configure(config, dry_run: true)
+  confs = []
+  masked_element = config.to_masked_element
+  masked_element.elements.each{|e|
+    confs << e.to_s
+  }
+  puts confs.join
+end

data/lib/fluent/plugin/calyptia_monitoring_buffer_ext.rb

@@ -0,0 +1,87 @@
+#
+# fluent-plugin-calyptia-monitoring
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/in_monitor_agent'
+
+module Fluent::Plugin
+  class CalyptiaMonitoringBufferExtInput < MonitorAgentInput
+    CALYPTIA_PLUGIN_BUFFER_METRIC_INFO = {
+      'buffer_total_queued_size' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.total_queued_size_metrics.cmetrics.to_msgpack
+      },
+      'buffer_stage_length' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.stage_length_metrics.cmetrics.to_msgpack
+      },
+      'buffer_stage_byte_size' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.stage_size_metrics.cmetrics.to_msgpack
+      },
+      'buffer_queue_length' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.queue_length_metrics.cmetrics.to_msgpack
+      },
+      'buffer_queue_byte_size' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.queue_size_metrics.cmetrics.to_msgpack
+      },
+      'available_buffer_space_ratios' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.available_buffer_space_ratios_metrics.cmetrics.to_msgpack
+      },
+      'newest_timekey' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.newest_timekey_metrics.cmetrics.to_msgpack
+      },
+      'oldest_timekey' => ->() {
+        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        @buffer.oldest_timekey_metrics.cmetrics.to_msgpack
+      },
+    }
+
+    def get_monitor_info(pe, opts = {})
+      obj = {}
+
+      obj['metrics'] = get_plugin_metric(pe)
+
+      obj
+    end
+
+    def get_plugin_metric(pe)
+      # Nop for non output plugin
+      return {} if plugin_category(pe) != "output"
+
+      metrics = {}
+
+      if pe.respond_to?(:statistics)
+        # Force to update buffers' metrics values
+        pe.statistics
+      end
+
+      CALYPTIA_PLUGIN_BUFFER_METRIC_INFO.each_pair { |key, code|
+        begin
+          v = pe.instance_exec(&code)
+          unless v.nil?
+            metrics[key] = v
+          end
+        rescue
+        end
+      }
+
+      metrics
+    end
+  end
+end

data/lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb

@@ -37,12 +37,17 @@ module Fluent::Plugin
         ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
       end
 
+      def create_go_semver(version)
+        version.gsub(/.(?<prever>(rc|alpha|beta|pre))/,
+                     '-\k<prever>')
+      end
+
       def agent_metadata(current_config)
         metadata = {
           "name" => Socket.gethostname,
           "type" => "fluentd",
           "rawConfig" => current_config,
-          "version" => Fluent::VERSION,
+          "version" => create_go_semver(Fluent::VERSION),
           "edition" => "community".freeze,
         }
         if system_config.workers.to_i > 1

data/lib/fluent/plugin/calyptia_monitoring_ext.rb

@@ -27,9 +27,25 @@ module Fluent::Plugin
         @emit_records_metrics.cmetrics.to_msgpack
       },
       'retry_count' => ->(){
-        throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+        throw(:skip) if @num_errors_metrics.get.nil?
         @num_errors_metrics.cmetrics.to_msgpack
       },
+      'write_count' => ->(){
+        throw(:skip) if @write_count_metrics.get.nil?
+        @write_count_metrics.cmetrics.to_msgpack
+      },
+      'rollback_count' => ->(){
+        throw(:skip) if @rollback_count_metrics.get.nil?
+        @rollback_count_metrics.cmetrics.to_msgpack
+      },
+      'flush_time_count' => ->(){
+        throw(:skip) if @flush_time_count_metrics.get.nil?
+        @flush_time_count_metrics.cmetrics.to_msgpack
+      },
+      'slow_flush_count' => ->(){
+        throw(:skip) if @slow_flush_count_metrics.get.nil?
+        @slow_flush_count_metrics.cmetrics.to_msgpack
+      },
     }
 
     def get_monitor_info(pe, opts = {})

data/lib/fluent/plugin/calyptia_monitoring_machine_id.rb

@@ -56,8 +56,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from ioreg. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{o.strip}:#{@worker_id}"
+        "#{o.strip}:#{@worker_id}"
       end
     end
 
@@ -72,8 +71,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from #{DBUS_MACHINE_ID_PATH} or #{ETC_MACHINE_ID_PATH}. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
 
@@ -88,8 +86,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from Registry. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
   end

data/lib/fluent/plugin/in_calyptia_monitoring.rb

@@ -17,10 +17,11 @@ require 'net/http'
 require 'monitor'
 require 'time'
 require 'fluent/version'
-require 'fluent/config/element'
-require 'fluent/plugin/metrics'
+require 'fluent/env'
 require "fluent/plugin/input"
+require "serverengine"
 require_relative "calyptia_monitoring_ext"
+require_relative "calyptia_monitoring_buffer_ext"
 require_relative "calyptia_monitoring_calyptia_api_requester"
 
 module Fluent
@@ -31,7 +32,7 @@ module Fluent
       class CreateAgentError < Fluent::ConfigError; end
       class UpdateAgentError < Fluent::ConfigError; end
 
-      helpers :timer, :storage
+      helpers :timer, :storage, :child_process
 
       RPC_CONFIG_DUMP_ENDPOINT = "/api/config.getDump".freeze
       DEFAULT_STORAGE_TYPE = 'local'
@@ -52,6 +53,8 @@ module Fluent
         config_param :rate, :time, default: 30
         desc 'Setup pending metrics capacity size'
         config_param :pending_metrics_size, :size, default: DEFAULT_PENDING_METRICS_SIZE
+        desc 'Specify Fluentd config file path for RPC not to be available case'
+        config_param :fluentd_conf_path, :string, default: nil
       end
 
       def multi_workers_ready?
@@ -67,6 +70,7 @@ module Fluent
 
       def configure(conf)
         super
+
         config = conf.elements.select{|e| e.name == 'storage' }.first
         @storage_agent = storage_create(usage: 'calyptia_monitoring_agent', conf: config, default_type: DEFAULT_STORAGE_TYPE)
       end
@@ -83,6 +87,43 @@ module Fluent
         confs.join
       end
 
+      def get_masked_conf_from_conf_file
+        return "" unless File.exist?(@cloud_monitoring.fluentd_conf_path) # check file existence.
+
+        conf = ""
+        callback = ->(status) {
+          if status && status.success?
+            #nop
+          elsif status
+            log.warn "config dumper exits with error code", prog: prog, status: status.exitstatus, signal: status.termsig
+          else
+            log.warn "config dumper unexpectedly exits without exit status", prog: prog
+          end
+        }
+        spawn_command, arguments = if Fluent.windows?
+                          [::ServerEngine.ruby_bin_path, File.join(File.dirname(__FILE__), "calyptia_config_dumper.rb")]
+                        else
+                          [File.join(File.dirname(__FILE__), "calyptia_config_dumper.rb")]
+                        end
+
+        retval = child_process_execute(:exec_calyptia_config_dumper, spawn_command, arguments: arguments, immediate: true,
+                                       env: {"FLUENT_CONFIG_PATH" => @cloud_monitoring.fluentd_conf_path}, parallel: true, mode: [:read_with_stderr],
+                                       on_exit_callback: callback) do |io|
+          io.set_encoding(Encoding::ASCII_8BIT)
+          conf = io.read
+        end
+        unless retval.nil?
+          begin
+            Timeout.timeout(10) do
+              sleep 0.1 until !conf.empty?
+            end
+          rescue Timeout::Error
+            log.warn "cannot retrive configuration contents on #{@cloud_monitoring.fluentd_conf_path} within 10 seconds."
+          end
+        end
+        conf
+      end
+
       def start
         super
 
@@ -93,13 +134,16 @@ module Fluent
                            end
         raise Fluent::ConfigError, "cmetrics plugin should be used to collect metrics on Calyptia Cloud" unless enabled_cmetrics
         @monitor_agent = Fluent::Plugin::CalyptiaMonitoringExtInput.new
+        @monitor_agent_buffer = Fluent::Plugin::CalyptiaMonitoringBufferExtInput.new
         @api_requester = Fluent::Plugin::CalyptiaAPI::Requester.new(@cloud_monitoring.endpoint,
                                                                     @cloud_monitoring.api_key,
                                                                     log,
                                                                     fluentd_worker_id)
-        if check_config_sending_usability
-          @current_config = get_current_config_from_rpc
-        end
+        @current_config = if !@cloud_monitoring.fluentd_conf_path.nil?
+                            get_masked_conf_from_conf_file
+                          elsif check_config_sending_usability
+                            get_current_config_from_rpc
+                          end
 
         if @cloud_monitoring.rate < 30
           log.warn "High frequency events ingestion is not supported. Set up 30s as ingestion interval"
@@ -114,7 +158,7 @@ module Fluent
 
       def create_agent(current_config)
         code, agent, machine_id = @api_requester.create_agent(current_config)
-        if agent["error"].nil?
+        if code.to_s.start_with?("2")
           @storage_agent.put(:agent, agent)
           @storage_agent.put(:machine_id, machine_id)
           return true
@@ -222,6 +266,12 @@ module Fluent
             buffer += v
           end
         }
+        @monitor_agent_buffer.plugins_info_all(opts).each {|record|
+          metrics = record["metrics"]
+          metrics.each_pair do |k, v|
+            buffer += v
+          end
+        }
         if buffer.empty?
           log.debug "No initialized metrics is found. Trying to send cmetrics on the next tick."
         else

data/templates/calyptia-conf.erb

@@ -0,0 +1,28 @@
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics <%= @enable_input_metrics %>
+  enable_size_metrics <%= @enable_size_metrics %>
+<%- unless @disable_rpc -%>
+  rpc_endpoint <%= @rpc_endpoint %>
+  enable_get_dump <%= @enable_get_dump %>
+<%- end -%>
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+<%- if @endpoint -%>
+    endpoint <%= @endpoint %>
+<%- end -%>
+    api_key <%= @api_key %>
+<%- if @fluentd_conf_path -%>
+    fluentd_conf_path <%= @fluentd_conf_path %>
+<%- end -%>
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path <%= @storage_agent_token_dir %>/agent_state
+  </storage>
+</source>

data/test/command/test_config_generator.rb

@@ -0,0 +1,230 @@
+require 'helper'
+require 'tmpdir'
+require 'fluent/command/config_generator'
+
+class TestCalyptiaConfigGenerator < Test::Unit::TestCase
+  def stringio_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+    out.string.force_encoding('utf-8')
+  ensure
+    $stdout = STDOUT
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  sub_test_case "generate configuration" do
+    test "with only api_key" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_input_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-input-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics false
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and rpc-endpoint" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--rpc-endpoint", "localhost:24445"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint localhost:24445
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_size_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--enable-size-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics true
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and disable_get_dump" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-get-dump"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump false
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and storage_agent_token_dir" do
+      storage_dir = Dir.tmpdir
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--storage_agent_token_dir", storage_dir]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and fluentd_config_path" do
+      storage_dir = Dir.tmpdir
+      conf_dir = Dir.tmpdir
+      conf_path = File.join(conf_dir, "fluent.conf")
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--storage_agent_token_dir", storage_dir, "--fluentd-conf-path", conf_path]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+    fluentd_conf_path #{conf_path}
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+  end
+end

data/test/plugin/test_calyptia_monitoring_calyptia_api_requester.rb

@@ -0,0 +1,29 @@
+require "helper"
+require "logger"
+require "fluent/plugin/calyptia_monitoring_calyptia_api_requester"
+
+class CalyptiaMonitoringMachineIdTest < Test::Unit::TestCase
+  API_KEY = 'YOUR_API_KEY'.freeze
+  API_ENDPOINT = "https://cloud-api.calyptia.com".freeze
+
+  setup do
+    @log = Logger.new($stdout)
+    worker_id = [1,2,3,4,5,6,7,8,9,10,11].sample
+    @api_requester = Fluent::Plugin::CalyptiaAPI::Requester.new(API_ENDPOINT,
+                                                                API_KEY,
+                                                                @log,
+                                                                worker_id)
+  end
+
+  data("rc" => ["1.14.0.rc", "1.14.0-rc"],
+       "rc2" => ["1.14.0.rc2", "1.14.0-rc2"],
+       "alpha" => ["1.14.0.alpha", "1.14.0-alpha"],
+       "beta" => ["1.14.0.beta", "1.14.0-beta"],
+       "pre" => ["1.14.0.pre", "1.14.0-pre"],
+      )
+  def test_create_go_semver(data)
+    version, expected = data
+    actual = @api_requester.create_go_semver(version)
+    assert_equal expected, actual
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calyptia-monitoring
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc6
+  version: 0.1.0
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-02 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 1.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +68,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 1.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -89,7 +89,8 @@ dependencies:
 description: Monitoring Fluentd via Calyptia Cloud
 email:
 - hatake@calyptia.com
-executables: []
+executables:
+- calyptia-config-generator
 extensions: []
 extra_rdoc_files: []
 files:
@@ -98,12 +99,19 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/calyptia-config-generator
 - fluent-plugin-calyptia-monitoring.gemspec
+- lib/fluent/command/config_generator.rb
+- lib/fluent/plugin/calyptia_config_dumper.rb
+- lib/fluent/plugin/calyptia_monitoring_buffer_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb
 - lib/fluent/plugin/calyptia_monitoring_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_machine_id.rb
 - lib/fluent/plugin/in_calyptia_monitoring.rb
+- templates/calyptia-conf.erb
+- test/command/test_config_generator.rb
 - test/helper.rb
+- test/plugin/test_calyptia_monitoring_calyptia_api_requester.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb
 homepage: https://github.com/calyptia/fluent-plugin-calyptia-monitoring
@@ -121,15 +129,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Monitoring Fluentd via Calyptia Cloud
 test_files:
+- test/command/test_config_generator.rb
 - test/helper.rb
+- test/plugin/test_calyptia_monitoring_calyptia_api_requester.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb