fluent-plugin-calyptia-monitoring 0.1.0.rc3 → 0.1.0.rc7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ce44716243a51ce3400cdb88a452b7fec82e12642ea0f1e9cdc175e6d8768d0
-  data.tar.gz: 8f4e3cdce49b511820d265d175519f3a65517c38a7b409dec90745bd425741da
+  metadata.gz: 7cfa17ee6e7d7b29c255151a698bdc9c004e936a1760a2f39ece7b204627f868
+  data.tar.gz: '098ab10f65f463b7de64f1a30adb828877285ff4aa6482fd3fec6e02e1e2de59'
 SHA512:
-  metadata.gz: '008865155bdee8e274d53f504ec08f2f5e52e91f7de806f4600632066e2995918dd9f01b63e9b18f785e865f662ebf203c9715e1cd3c55eded5e8870855f95ff'
-  data.tar.gz: 88d3b9026053c6cf8feec03f186c870a01dbe898a159fd3d9f06fb6643d4194344fd18a2cf85e52712844a79c3f34fb1e0671d61842433a1ff0a34f0543137f3
+  metadata.gz: e545fd36229c0be2bfbcf2a60045033ae95e131e67bd5ee810a2b784a71a4ba707684bb4d8674e7d891565ef0fce59b4f46f72c21fc4e30e60b94bf9274a6673
+  data.tar.gz: d408f12b6c6a12ca21e402858703c57692b87c57a7571017e5c6beef30e019c15c1adea4f5def05711cdfef79c50c01286f839babec0501a033a43ab4e7f137a

data/README.md

@@ -56,6 +56,9 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 # If users want to use multi workers feature which corresponds to logical number of CPUs, please comment out this line.
 #  workers "#{require 'etc'; Etc.nprocessors}"
   enable_input_metrics true
+  # This record size measuring settings might impact for performance.
+  # Please be careful for high loaded environment to turn on.
+  enable_size_metrics true
   <metrics>
     @type cmetrics
   </metrics>
@@ -81,6 +84,28 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 </source>
 ```
 
+## Calyptia Monitoring API config generator
+
+Usage:
+
+```
+Usage: calyptia-config-generator api_key [options]
+
+Output plugin config definitions
+
+Arguments:
+	api_key: Specify your API_KEY
+
+Options:
+        --endpoint URL               API Endpoint URL (default: nil, and if omitted, using default endpoint)
+        --rpc-endpoint URL           Specify RPC Endpoint URL (default: 127.0.0.1:24444)
+        --disable-input-metrics      Disable Input plugin metrics. Input metrics is enabled by default
+        --enable-size-metrics        Enable event size metrics. Size metrics is disabled by default.
+        --disable-get-dump           Disable RPC getDump procedure. getDump is enabled by default.
+        --storage-agent-token-dir DIR
+                                     Specify accesible storage token dir. (default: /path/to/accesible/dir)
+```
+
 ## Copyright
 
 * Copyright(c) 2021- Hiroshi Hatake

data/bin/calyptia-config-generator

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+require 'fluent/command/config_generator'
+
+CalyptiaConfigGenerator.new.call

data/fluent-plugin-calyptia-monitoring.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-calyptia-monitoring"
-  spec.version = "0.1.0.rc3"
+  spec.version = "0.1.0.rc7"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["hatake@calyptia.com"]
 
@@ -24,5 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "test-unit", "~> 3.3"
   spec.add_runtime_dependency "fluentd", [">= 1.13.0", "< 2"]
-  spec.add_runtime_dependency "fluent-plugin-metrics-cmetrics", ">= 0.1.0.rc2"
+  spec.add_runtime_dependency "fluent-plugin-metrics-cmetrics", ">= 0.1.0.rc3"
 end

data/lib/fluent/command/config_generator.rb

@@ -0,0 +1,136 @@
+#
+# fluent-plugin-calyptia-monitoring
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+require 'fluent/version'
+
+class CalyptiaConfigGenerator
+  def initialize(argv = ARGV)
+    @argv = argv
+    @api_key = nil
+    @endpoint = nil
+    @enable_input_metrics = true
+    @enable_size_metrics = false
+    @enable_get_dump = true
+    @rpc_endpoint = "127.0.0.1:24444"
+    @storage_agent_token_dir = default_storage_dir
+
+    prepare_option_parser
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  def call
+    parse_options!
+
+    puts dump_configuration_for_calyptia
+  end
+
+  def dump_configuration_for_calyptia
+    dumped = ""
+    template = template_path("calyptia-conf.erb").read
+
+    dumped <<
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(template, trim_mode: "-")
+      else
+        ERB.new(template, nil, "-")
+      end.result(binding)
+    dumped
+  end
+  private
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: #{$0} api_key [options]
+
+Output plugin config definitions
+
+Arguments:
+\tapi_key: Specify your API_KEY
+
+Options:
+BANNER
+    @parser.on("--endpoint URL", "API Endpoint URL (default: nil, and if omitted, using default endpoint)") do |s|
+      @endpoint = s
+    end
+    @parser.on("--rpc-endpoint URL", "Specify RPC Endpoint URL (default: 127.0.0.1:24444)") do |s|
+      @rpc_endpoint = s
+    end
+    @parser.on("--disable-input-metrics", "Disable Input plugin metrics. Input metrics is enabled by default") do
+      @enable_input_metrics = false
+    end
+    @parser.on("--enable-size-metrics", "Enable event size metrics. Size metrics is disabled by default.") do
+      @enable_size_metrics = true
+    end
+    @parser.on("--disable-get-dump", "Disable RPC getDump procedure. getDump is enabled by default.") do
+      @enable_get_dump = false
+    end
+    @parser.on("--storage-agent-token-dir DIR", "Specify accesible storage token dir. (default: #{default_storage_dir})") do |s|
+      @storage_agent_token_dir = s
+    end
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify api_key" unless @argv.size == 1
+
+    host, port = @rpc_endpoint.split(':')
+    if @enable_get_dump && (!host || !port)
+      puts "Error: invalid rpc_endpoint format. Specify `host:port' form."
+      exit(false)
+    end
+
+    @api_key, = @argv
+    @options = {
+      api_key: @api_key,
+      endpoint: @endpoint,
+      rpc_endpoint: @rpc_endpoint,
+      input_metrics: @enable_input_metrics,
+      size_metrics: @enable_size_metrics,
+      enable_get_dump: @enable_get_dump,
+      storage_agent_token_dir: @storage_agent_token_dir,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/#{name}").realpath
+  end
+end

data/lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb

@@ -51,10 +51,10 @@ module Fluent::Plugin
         metadata
       end
 
-      # POST /api/v1/agents
+      # POST /v1/agents
       # Authorization: X-Project-Token
       def create_agent(current_config)
-        url = URI("#{@endpoint}/api/v1/agents")
+        url = URI("#{@endpoint}/v1/agents")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)
@@ -75,10 +75,10 @@ module Fluent::Plugin
         return [response.code, agent, machine_id]
       end
 
-      # PATCH /api/v1/agents/:agent_id
+      # PATCH /v1/agents/:agent_id
       # Authorization: X-Agent-Token
       def update_agent(current_config, agent, machine_id)
-        url = URI("#{@endpoint}/api/v1/agents/#{agent['id']}")
+        url = URI("#{@endpoint}/v1/agents/#{agent['id']}")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)
@@ -100,10 +100,10 @@ module Fluent::Plugin
         return [response.code, body]
       end
 
-      # POST /api/v1/agents/:agent_id/metrics
+      # POST /v1/agents/:agent_id/metrics
       # Authorization: X-Agent-Token
       def add_metrics(metrics, agent_token, agent_id)
-        url = URI("#{@endpoint}/api/v1/agents/#{agent_id}/metrics")
+        url = URI("#{@endpoint}/v1/agents/#{agent_id}/metrics")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)

data/lib/fluent/plugin/calyptia_monitoring_ext.rb

@@ -19,11 +19,11 @@ module Fluent::Plugin
   class CalyptiaMonitoringExtInput < MonitorAgentInput
     CALYPTIA_PLUGIN_METRIC_INFO = {
       'emit_size' => ->(){
-        throw(:skip) if @emit_size_metrics.get(self.plugin_id).nil?
+        throw(:skip) if @emit_size_metrics.get.nil?
         @emit_size_metrics.cmetrics.to_msgpack
       },
       'emit_records' => ->(){
-        throw(:skip) if @emit_records_metrics.get(self.plugin_id).nil?
+        throw(:skip) if @emit_records_metrics.get.nil?
         @emit_records_metrics.cmetrics.to_msgpack
       },
       'retry_count' => ->(){

data/lib/fluent/plugin/calyptia_monitoring_machine_id.rb

@@ -56,8 +56,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from ioreg. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{o.strip}:#{@worker_id}"
+        "#{o.strip}:#{@worker_id}"
       end
     end
 
@@ -65,15 +64,14 @@ module Fluent::Plugin
       machine_id = ""
       begin
         machine_id = File.read(DBUS_MACHINE_ID_PATH).strip
-      rescue Errno::NOENT
+      rescue Errno::ENOENT
         machine_id = File.read(ETC_MACHINE_ID_PATH).strip rescue ""
       end
       if machine_id.empty?
         @log.info "MachineID is not retrived from #{DBUS_MACHINE_ID_PATH} or #{ETC_MACHINE_ID_PATH}. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
 
@@ -88,8 +86,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from Registry. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
   end

data/lib/fluent/plugin/in_calyptia_monitoring.rb

@@ -45,7 +45,7 @@ module Fluent
 
       config_section :cloud_monitoring, multi: false, required: true do
         desc 'The endpoint for Monitoring API HTTP request, e.g. http://example.com/api'
-        config_param :endpoint, :string, default: "https://cloud-monitoring-api-dev.fluentbit.io"
+        config_param :endpoint, :string, default: "https://cloud-api.calyptia.com"
         desc 'The API KEY for Monitoring API HTTP request'
         config_param :api_key, :string, secret: true
         desc 'Emit monitoring values interval. (minimum interval is 30 seconds.)'
@@ -114,7 +114,7 @@ module Fluent
 
       def create_agent(current_config)
         code, agent, machine_id = @api_requester.create_agent(current_config)
-        if agent["error"].nil?
+        if code.to_s.start_with?("2")
           @storage_agent.put(:agent, agent)
           @storage_agent.put(:machine_id, machine_id)
           return true

data/templates/calyptia-conf.erb

@@ -0,0 +1,23 @@
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics <%= @enable_input_metrics %>
+  enable_size_metrics <%= @enable_size_metrics %>
+  rpc_endpoint <%= @rpc_endpoint %>
+  enable_get_dump <%= @enable_get_dump %>
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+<%- if @endpoint -%>
+    endpoint <%= @endpoint %>
+<%- end -%>
+    api_key <%= @api_key %>
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path <%= @storage_agent_token_dir %>/agent_state
+  </storage>
+</source>

data/test/command/test_config_generator.rb

@@ -0,0 +1,199 @@
+require 'helper'
+require 'tmpdir'
+require 'fluent/command/config_generator'
+
+class TestCalyptiaConfigGenerator < Test::Unit::TestCase
+  def stringio_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+    out.string.force_encoding('utf-8')
+  ensure
+    $stdout = STDOUT
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  sub_test_case "generate configuration" do
+    test "with only api_key" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_input_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-input-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics false
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and rpc-endpoint" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--rpc-endpoint", "localhost:24445"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint localhost:24445
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_size_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--enable-size-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics true
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and disable_get_dump" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-get-dump"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump false
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and storage_agent_token_dir" do
+      storage_dir = Dir.tmpdir
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--storage_agent_token_dir", storage_dir]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calyptia-monitoring
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc3
+  version: 0.1.0.rc7
 platform: ruby
 authors:
 - Hiroshi Hatake
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-27 00:00:00.000000000 Z
+date: 2021-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -78,18 +78,19 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0.rc2
+        version: 0.1.0.rc3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0.rc2
+        version: 0.1.0.rc3
 description: Monitoring Fluentd via Calyptia Cloud
 email:
 - hatake@calyptia.com
-executables: []
+executables:
+- calyptia-config-generator
 extensions: []
 extra_rdoc_files: []
 files:
@@ -98,11 +99,15 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/calyptia-config-generator
 - fluent-plugin-calyptia-monitoring.gemspec
+- lib/fluent/command/config_generator.rb
 - lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb
 - lib/fluent/plugin/calyptia_monitoring_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_machine_id.rb
 - lib/fluent/plugin/in_calyptia_monitoring.rb
+- templates/calyptia-conf.erb
+- test/command/test_config_generator.rb
 - test/helper.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb
@@ -110,7 +115,7 @@ homepage: https://github.com/calyptia/fluent-plugin-calyptia-monitoring
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -125,11 +130,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Monitoring Fluentd via Calyptia Cloud
 test_files:
+- test/command/test_config_generator.rb
 - test/helper.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb