fluent-plugin-calyptia-monitoring 0.1.0.rc4 → 0.1.0.rc8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c6fb8e1a464d6431e9e90ec1baabdcb99daaff5d166d2d92da2d756f0f08968
-  data.tar.gz: 75f9eba0d35c2a56f47866cbb8c376ecacdc30b79b707e03481bfeed067a5e1f
+  metadata.gz: 071d15721bdabcee712ce74be5d1f7125458e97f79f88c8e1b30927d353a9f04
+  data.tar.gz: 8704372a766aeb87d7993cf7e42ff0b2a07b6927398a25ee125bda03d035315d
 SHA512:
-  metadata.gz: 7c5332bd91f9bf937d563a71812256c2870cd56b605d55abae77cb986cafc5f023584b3e2257d1fca307e75372b68175da0001c3f7158566f8fc3765925bd08e
-  data.tar.gz: 8d56f117039feef859229e84aacdb92b8f0953ed7d20831a947c2d4a84ec62f885c4ef6e0ba00addf2f5c85450dc05a114c327f76468c84fa3f05c1a26720db9
+  metadata.gz: 5c8c50b693112a46959ee857d356a53957957fd1823eb4d98773765e7f3141535e0c3aac3e17e4cba80d930aea0f8b33e5174105955dfef0cb2d01a341ad05d1
+  data.tar.gz: 59a2cb53277e5bc524f5e18a534764252a523d94c3ce5b9e3fdd46214f5a61bb191398255fbcd99f858dc335ebd22f3537ba4b23fc76734bfb2720381dc7aa79

data/README.md

@@ -56,6 +56,9 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 # If users want to use multi workers feature which corresponds to logical number of CPUs, please comment out this line.
 #  workers "#{require 'etc'; Etc.nprocessors}"
   enable_input_metrics true
+  # This record size measuring settings might impact for performance.
+  # Please be careful for high loaded environment to turn on.
+  enable_size_metrics true
   <metrics>
     @type cmetrics
   </metrics>
@@ -81,6 +84,28 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 </source>
 ```
 
+## Calyptia Monitoring API config generator
+
+Usage:
+
+```
+Usage: calyptia-config-generator api_key [options]
+
+Output plugin config definitions
+
+Arguments:
+	api_key: Specify your API_KEY
+
+Options:
+        --endpoint URL               API Endpoint URL (default: nil, and if omitted, using default endpoint)
+        --rpc-endpoint URL           Specify RPC Endpoint URL (default: 127.0.0.1:24444)
+        --disable-input-metrics      Disable Input plugin metrics. Input metrics is enabled by default
+        --enable-size-metrics        Enable event size metrics. Size metrics is disabled by default.
+        --disable-get-dump           Disable RPC getDump procedure. getDump is enabled by default.
+        --storage-agent-token-dir DIR
+                                     Specify accesible storage token dir. (default: /path/to/accesible/dir)
+```
+
 ## Copyright
 
 * Copyright(c) 2021- Hiroshi Hatake

data/bin/calyptia-config-generator

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+require 'fluent/command/config_generator'
+
+CalyptiaConfigGenerator.new.call

data/fluent-plugin-calyptia-monitoring.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-calyptia-monitoring"
-  spec.version = "0.1.0.rc4"
+  spec.version = "0.1.0.rc8"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["hatake@calyptia.com"]
 

data/lib/fluent/command/config_generator.rb

@@ -0,0 +1,136 @@
+#
+# fluent-plugin-calyptia-monitoring
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+require 'fluent/version'
+
+class CalyptiaConfigGenerator
+  def initialize(argv = ARGV)
+    @argv = argv
+    @api_key = nil
+    @endpoint = nil
+    @enable_input_metrics = true
+    @enable_size_metrics = false
+    @enable_get_dump = true
+    @rpc_endpoint = "127.0.0.1:24444"
+    @storage_agent_token_dir = default_storage_dir
+
+    prepare_option_parser
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  def call
+    parse_options!
+
+    puts dump_configuration_for_calyptia
+  end
+
+  def dump_configuration_for_calyptia
+    dumped = ""
+    template = template_path("calyptia-conf.erb").read
+
+    dumped <<
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(template, trim_mode: "-")
+      else
+        ERB.new(template, nil, "-")
+      end.result(binding)
+    dumped
+  end
+  private
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: #{$0} api_key [options]
+
+Generate Calyptia monitoring plugin config definitions
+
+Arguments:
+\tapi_key: Specify your API_KEY
+
+Options:
+BANNER
+    @parser.on("--endpoint URL", "API Endpoint URL (default: nil, and if omitted, using default endpoint)") do |s|
+      @endpoint = s
+    end
+    @parser.on("--rpc-endpoint URL", "Specify RPC Endpoint URL (default: 127.0.0.1:24444)") do |s|
+      @rpc_endpoint = s
+    end
+    @parser.on("--disable-input-metrics", "Disable Input plugin metrics. Input metrics is enabled by default") do
+      @enable_input_metrics = false
+    end
+    @parser.on("--enable-size-metrics", "Enable event size metrics. Size metrics is disabled by default.") do
+      @enable_size_metrics = true
+    end
+    @parser.on("--disable-get-dump", "Disable RPC getDump procedure. getDump is enabled by default.") do
+      @enable_get_dump = false
+    end
+    @parser.on("--storage-agent-token-dir DIR", "Specify accesible storage token dir. (default: #{default_storage_dir})") do |s|
+      @storage_agent_token_dir = s
+    end
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify api_key" unless @argv.size == 1
+
+    host, port = @rpc_endpoint.split(':')
+    if @enable_get_dump && (!host || !port)
+      puts "Error: invalid rpc_endpoint format. Specify `host:port' form."
+      exit(false)
+    end
+
+    @api_key, = @argv
+    @options = {
+      api_key: @api_key,
+      endpoint: @endpoint,
+      rpc_endpoint: @rpc_endpoint,
+      input_metrics: @enable_input_metrics,
+      size_metrics: @enable_size_metrics,
+      enable_get_dump: @enable_get_dump,
+      storage_agent_token_dir: @storage_agent_token_dir,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/#{name}").realpath
+  end
+end

data/lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb

@@ -51,10 +51,10 @@ module Fluent::Plugin
         metadata
       end
 
-      # POST /api/v1/agents
+      # POST /v1/agents
       # Authorization: X-Project-Token
       def create_agent(current_config)
-        url = URI("#{@endpoint}/api/v1/agents")
+        url = URI("#{@endpoint}/v1/agents")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)
@@ -75,10 +75,10 @@ module Fluent::Plugin
         return [response.code, agent, machine_id]
       end
 
-      # PATCH /api/v1/agents/:agent_id
+      # PATCH /v1/agents/:agent_id
       # Authorization: X-Agent-Token
       def update_agent(current_config, agent, machine_id)
-        url = URI("#{@endpoint}/api/v1/agents/#{agent['id']}")
+        url = URI("#{@endpoint}/v1/agents/#{agent['id']}")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)
@@ -100,10 +100,10 @@ module Fluent::Plugin
         return [response.code, body]
       end
 
-      # POST /api/v1/agents/:agent_id/metrics
+      # POST /v1/agents/:agent_id/metrics
       # Authorization: X-Agent-Token
       def add_metrics(metrics, agent_token, agent_id)
-        url = URI("#{@endpoint}/api/v1/agents/#{agent_id}/metrics")
+        url = URI("#{@endpoint}/v1/agents/#{agent_id}/metrics")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)

data/lib/fluent/plugin/calyptia_monitoring_machine_id.rb

@@ -56,8 +56,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from ioreg. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{o.strip}:#{@worker_id}"
+        "#{o.strip}:#{@worker_id}"
       end
     end
 
@@ -65,15 +64,14 @@ module Fluent::Plugin
       machine_id = ""
       begin
         machine_id = File.read(DBUS_MACHINE_ID_PATH).strip
-      rescue Errno::NOENT
+      rescue Errno::ENOENT
         machine_id = File.read(ETC_MACHINE_ID_PATH).strip rescue ""
       end
       if machine_id.empty?
         @log.info "MachineID is not retrived from #{DBUS_MACHINE_ID_PATH} or #{ETC_MACHINE_ID_PATH}. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
 
@@ -88,8 +86,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from Registry. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
   end

data/lib/fluent/plugin/in_calyptia_monitoring.rb

@@ -45,7 +45,7 @@ module Fluent
 
       config_section :cloud_monitoring, multi: false, required: true do
         desc 'The endpoint for Monitoring API HTTP request, e.g. http://example.com/api'
-        config_param :endpoint, :string, default: "https://cloud-monitoring-api-dev.fluentbit.io"
+        config_param :endpoint, :string, default: "https://cloud-api.calyptia.com"
         desc 'The API KEY for Monitoring API HTTP request'
         config_param :api_key, :string, secret: true
         desc 'Emit monitoring values interval. (minimum interval is 30 seconds.)'
@@ -114,7 +114,7 @@ module Fluent
 
       def create_agent(current_config)
         code, agent, machine_id = @api_requester.create_agent(current_config)
-        if agent["error"].nil?
+        if code.to_s.start_with?("2")
           @storage_agent.put(:agent, agent)
           @storage_agent.put(:machine_id, machine_id)
           return true

data/templates/calyptia-conf.erb

@@ -0,0 +1,23 @@
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics <%= @enable_input_metrics %>
+  enable_size_metrics <%= @enable_size_metrics %>
+  rpc_endpoint <%= @rpc_endpoint %>
+  enable_get_dump <%= @enable_get_dump %>
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+<%- if @endpoint -%>
+    endpoint <%= @endpoint %>
+<%- end -%>
+    api_key <%= @api_key %>
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path <%= @storage_agent_token_dir %>/agent_state
+  </storage>
+</source>

data/test/command/test_config_generator.rb

@@ -0,0 +1,199 @@
+require 'helper'
+require 'tmpdir'
+require 'fluent/command/config_generator'
+
+class TestCalyptiaConfigGenerator < Test::Unit::TestCase
+  def stringio_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+    out.string.force_encoding('utf-8')
+  ensure
+    $stdout = STDOUT
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  sub_test_case "generate configuration" do
+    test "with only api_key" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_input_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-input-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics false
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and rpc-endpoint" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--rpc-endpoint", "localhost:24445"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint localhost:24445
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_size_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--enable-size-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics true
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and disable_get_dump" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-get-dump"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump false
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and storage_agent_token_dir" do
+      storage_dir = Dir.tmpdir
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--storage_agent_token_dir", storage_dir]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calyptia-monitoring
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc4
+  version: 0.1.0.rc8
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -89,7 +89,8 @@ dependencies:
 description: Monitoring Fluentd via Calyptia Cloud
 email:
 - hatake@calyptia.com
-executables: []
+executables:
+- calyptia-config-generator
 extensions: []
 extra_rdoc_files: []
 files:
@@ -98,11 +99,15 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/calyptia-config-generator
 - fluent-plugin-calyptia-monitoring.gemspec
+- lib/fluent/command/config_generator.rb
 - lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb
 - lib/fluent/plugin/calyptia_monitoring_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_machine_id.rb
 - lib/fluent/plugin/in_calyptia_monitoring.rb
+- templates/calyptia-conf.erb
+- test/command/test_config_generator.rb
 - test/helper.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb
@@ -130,6 +135,7 @@ signing_key:
 specification_version: 4
 summary: Monitoring Fluentd via Calyptia Cloud
 test_files:
+- test/command/test_config_generator.rb
 - test/helper.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb