fluent-plugin-calyptia-monitoring 0.1.0.rc5 → 0.1.0.rc9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45f4c392adfd8ddda9ee6c9387f0092472dec1a734274308de4ce6e325b5bc40
-  data.tar.gz: 0ba927c3eac167f867342ddfea86f885a2aa4c2f3ea099faa9dbad22fa4af9c1
+  metadata.gz: e42e4240470d5d46bcf2d098699d82d1d0eb6440f88fd89ff004d7c79b5789f7
+  data.tar.gz: 7f164695485c5202e003a31defc1b5a4022b67ed25ac71550011eb6400946f43
 SHA512:
-  metadata.gz: 3d147883bb7b483b9b141713e05e5c2ea63f8b69b7680acd0fa931e84c8c55028d6ba6a8a6876e898e1d72b9b3a446df1a5e2875f022c6d070a94f7c49930fa9
-  data.tar.gz: a9ccd6a159b2c6efdcf330cfa245b7b46b859bb5db476dc5e24e9dcecf9ee3a8439b7da28f0755176f1d52c42c7232230c9fca009e2032042a0b9cfcf28567db
+  metadata.gz: ce39e348d421451fda13460ec7072d35e6e79df6c5bc74dfb48fb8f5d9237ff3c4a97bee8034d6e69ba76540b0c375b0cc599b78606b18adecb146a8b316d24a
+  data.tar.gz: 171bf4f4c6230b68174aafb76d042059956f0404294157c06b6a763ffb4cd2d726fb93cf9dc9956aa598c12ed5cda1aebdba1fac0a8a7af293907d230e91ef18

data/README.md

@@ -84,6 +84,28 @@ And enabling RPC and configDump endpoint is required if sending Fluentd configur
 </source>
 ```
 
+## Calyptia Monitoring API config generator
+
+Usage:
+
+```
+Usage: calyptia-config-generator api_key [options]
+
+Generate Calyptia monitoring plugin config definitions
+
+Arguments:
+	api_key: Specify your API_KEY
+
+Options:
+        --endpoint URL               API Endpoint URL (default: nil, and if omitted, using default endpoint)
+        --rpc-endpoint URL           Specify RPC Endpoint URL (default: 127.0.0.1:24444)
+        --disable-input-metrics      Disable Input plugin metrics. Input metrics is enabled by default
+        --enable-size-metrics        Enable event size metrics. Size metrics is disabled by default.
+        --disable-get-dump           Disable RPC getDump procedure. getDump is enabled by default.
+        --storage-agent-token-dir DIR
+                                     Specify accesible storage token dir. (default: /path/to/accesible/dir)
+```
+
 ## Copyright
 
 * Copyright(c) 2021- Hiroshi Hatake

data/bin/calyptia-config-generator

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+require 'fluent/command/config_generator'
+
+CalyptiaConfigGenerator.new.call

data/fluent-plugin-calyptia-monitoring.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-calyptia-monitoring"
-  spec.version = "0.1.0.rc5"
+  spec.version = "0.1.0.rc9"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["hatake@calyptia.com"]
 
@@ -23,6 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.2.15"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "test-unit", "~> 3.3"
-  spec.add_runtime_dependency "fluentd", [">= 1.13.0", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 1.14.0.rc", "< 2"]
   spec.add_runtime_dependency "fluent-plugin-metrics-cmetrics", ">= 0.1.0.rc3"
 end

data/lib/fluent/command/config_generator.rb

@@ -0,0 +1,136 @@
+#
+# fluent-plugin-calyptia-monitoring
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+require 'fluent/version'
+
+class CalyptiaConfigGenerator
+  def initialize(argv = ARGV)
+    @argv = argv
+    @api_key = nil
+    @endpoint = nil
+    @enable_input_metrics = true
+    @enable_size_metrics = false
+    @enable_get_dump = true
+    @rpc_endpoint = "127.0.0.1:24444"
+    @storage_agent_token_dir = default_storage_dir
+
+    prepare_option_parser
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  def call
+    parse_options!
+
+    puts dump_configuration_for_calyptia
+  end
+
+  def dump_configuration_for_calyptia
+    dumped = ""
+    template = template_path("calyptia-conf.erb").read
+
+    dumped <<
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(template, trim_mode: "-")
+      else
+        ERB.new(template, nil, "-")
+      end.result(binding)
+    dumped
+  end
+  private
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: #{$0} api_key [options]
+
+Generate Calyptia monitoring plugin config definitions
+
+Arguments:
+\tapi_key: Specify your API_KEY
+
+Options:
+BANNER
+    @parser.on("--endpoint URL", "API Endpoint URL (default: nil, and if omitted, using default endpoint)") do |s|
+      @endpoint = s
+    end
+    @parser.on("--rpc-endpoint URL", "Specify RPC Endpoint URL (default: 127.0.0.1:24444)") do |s|
+      @rpc_endpoint = s
+    end
+    @parser.on("--disable-input-metrics", "Disable Input plugin metrics. Input metrics is enabled by default") do
+      @enable_input_metrics = false
+    end
+    @parser.on("--enable-size-metrics", "Enable event size metrics. Size metrics is disabled by default.") do
+      @enable_size_metrics = true
+    end
+    @parser.on("--disable-get-dump", "Disable RPC getDump procedure. getDump is enabled by default.") do
+      @enable_get_dump = false
+    end
+    @parser.on("--storage-agent-token-dir DIR", "Specify accesible storage token dir. (default: #{default_storage_dir})") do |s|
+      @storage_agent_token_dir = s
+    end
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify api_key" unless @argv.size == 1
+
+    host, port = @rpc_endpoint.split(':')
+    if @enable_get_dump && (!host || !port)
+      puts "Error: invalid rpc_endpoint format. Specify `host:port' form."
+      exit(false)
+    end
+
+    @api_key, = @argv
+    @options = {
+      api_key: @api_key,
+      endpoint: @endpoint,
+      rpc_endpoint: @rpc_endpoint,
+      input_metrics: @enable_input_metrics,
+      size_metrics: @enable_size_metrics,
+      enable_get_dump: @enable_get_dump,
+      storage_agent_token_dir: @storage_agent_token_dir,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/#{name}").realpath
+  end
+end

data/lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb

@@ -37,12 +37,17 @@ module Fluent::Plugin
         ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
       end
 
+      def create_go_semver(version)
+        version.gsub(/.(?<prever>(rc|alpha|beta|pre))/,
+                     '-\k<prever>')
+      end
+
       def agent_metadata(current_config)
         metadata = {
           "name" => Socket.gethostname,
           "type" => "fluentd",
           "rawConfig" => current_config,
-          "version" => Fluent::VERSION,
+          "version" => create_go_semver(Fluent::VERSION),
           "edition" => "community".freeze,
         }
         if system_config.workers.to_i > 1
@@ -51,10 +56,10 @@ module Fluent::Plugin
         metadata
       end
 
-      # POST /api/v1/agents
+      # POST /v1/agents
       # Authorization: X-Project-Token
       def create_agent(current_config)
-        url = URI("#{@endpoint}/api/v1/agents")
+        url = URI("#{@endpoint}/v1/agents")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)
@@ -75,10 +80,10 @@ module Fluent::Plugin
         return [response.code, agent, machine_id]
       end
 
-      # PATCH /api/v1/agents/:agent_id
+      # PATCH /v1/agents/:agent_id
       # Authorization: X-Agent-Token
       def update_agent(current_config, agent, machine_id)
-        url = URI("#{@endpoint}/api/v1/agents/#{agent['id']}")
+        url = URI("#{@endpoint}/v1/agents/#{agent['id']}")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)
@@ -100,10 +105,10 @@ module Fluent::Plugin
         return [response.code, body]
       end
 
-      # POST /api/v1/agents/:agent_id/metrics
+      # POST /v1/agents/:agent_id/metrics
       # Authorization: X-Agent-Token
       def add_metrics(metrics, agent_token, agent_id)
-        url = URI("#{@endpoint}/api/v1/agents/#{agent_id}/metrics")
+        url = URI("#{@endpoint}/v1/agents/#{agent_id}/metrics")
 
         https = if proxy = proxies
                   proxy_uri = URI.parse(proxy)

data/lib/fluent/plugin/calyptia_monitoring_machine_id.rb

@@ -56,8 +56,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from ioreg. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{o.strip}:#{@worker_id}"
+        "#{o.strip}:#{@worker_id}"
       end
     end
 
@@ -72,8 +71,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from #{DBUS_MACHINE_ID_PATH} or #{ETC_MACHINE_ID_PATH}. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
 
@@ -88,8 +86,7 @@ module Fluent::Plugin
         @log.info "MachineID is not retrived from Registry. Using UUID instead."
         "#{SecureRandom.uuid}:#{@worker_id}"
       else
-        # TODO: The prefix should be removed?
-        "#{SecureRandom.hex(10)}_#{machine_id}:#{@worker_id}"
+        "#{machine_id}:#{@worker_id}"
       end
     end
   end

data/lib/fluent/plugin/in_calyptia_monitoring.rb

@@ -45,7 +45,7 @@ module Fluent
 
       config_section :cloud_monitoring, multi: false, required: true do
         desc 'The endpoint for Monitoring API HTTP request, e.g. http://example.com/api'
-        config_param :endpoint, :string, default: "https://cloud-monitoring-api-dev.fluentbit.io"
+        config_param :endpoint, :string, default: "https://cloud-api.calyptia.com"
         desc 'The API KEY for Monitoring API HTTP request'
         config_param :api_key, :string, secret: true
         desc 'Emit monitoring values interval. (minimum interval is 30 seconds.)'
@@ -114,7 +114,7 @@ module Fluent
 
       def create_agent(current_config)
         code, agent, machine_id = @api_requester.create_agent(current_config)
-        if agent["error"].nil?
+        if code.to_s.start_with?("2")
           @storage_agent.put(:agent, agent)
           @storage_agent.put(:machine_id, machine_id)
           return true

data/templates/calyptia-conf.erb

@@ -0,0 +1,23 @@
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics <%= @enable_input_metrics %>
+  enable_size_metrics <%= @enable_size_metrics %>
+  rpc_endpoint <%= @rpc_endpoint %>
+  enable_get_dump <%= @enable_get_dump %>
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+<%- if @endpoint -%>
+    endpoint <%= @endpoint %>
+<%- end -%>
+    api_key <%= @api_key %>
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path <%= @storage_agent_token_dir %>/agent_state
+  </storage>
+</source>

data/test/command/test_config_generator.rb

@@ -0,0 +1,199 @@
+require 'helper'
+require 'tmpdir'
+require 'fluent/command/config_generator'
+
+class TestCalyptiaConfigGenerator < Test::Unit::TestCase
+  def stringio_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+    out.string.force_encoding('utf-8')
+  ensure
+    $stdout = STDOUT
+  end
+
+  def default_storage_dir
+    if Fluent.windows?
+      "C:/path/to/accesible/dir"
+    else
+      "/path/to/accesible/dir"
+    end
+  end
+
+  sub_test_case "generate configuration" do
+    test "with only api_key" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_input_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-input-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics false
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and rpc-endpoint" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--rpc-endpoint", "localhost:24445"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint localhost:24445
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and enable_size_metrics" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--enable-size-metrics"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics true
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and disable_get_dump" do
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--disable-get-dump"]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump false
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{default_storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "with api_key and storage_agent_token_dir" do
+      storage_dir = Dir.tmpdir
+      dumped_config = capture_stdout do
+        CalyptiaConfigGenerator.new(["YOUR_API_KEY", "--storage_agent_token_dir", storage_dir]).call
+      end
+      expected =<<TEXT
+<system>
+  <metrics>
+    @type cmetrics
+  </metrics>
+  enable_input_metrics true
+  enable_size_metrics false
+  rpc_endpoint 127.0.0.1:24444
+  enable_get_dump true
+<system>
+<source>
+  @type calyptia_monitoring
+  @id input_caplyptia_monitoring
+  <cloud_monitoring>
+    api_key YOUR_API_KEY
+  </cloud_monitoring>
+  <storage>
+    @type local
+    path #{storage_dir}/agent_state
+  </storage>
+</source>
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+  end
+end

data/test/plugin/test_calyptia_monitoring_calyptia_api_requester.rb

@@ -0,0 +1,29 @@
+require "helper"
+require "logger"
+require "fluent/plugin/calyptia_monitoring_calyptia_api_requester"
+
+class CalyptiaMonitoringMachineIdTest < Test::Unit::TestCase
+  API_KEY = 'YOUR_API_KEY'.freeze
+  API_ENDPOINT = "https://cloud-api.calyptia.com".freeze
+
+  setup do
+    @log = Logger.new($stdout)
+    worker_id = [1,2,3,4,5,6,7,8,9,10,11].sample
+    @api_requester = Fluent::Plugin::CalyptiaAPI::Requester.new(API_ENDPOINT,
+                                                                API_KEY,
+                                                                @log,
+                                                                worker_id)
+  end
+
+  data("rc" => ["1.14.0.rc", "1.14.0-rc"],
+       "rc2" => ["1.14.0.rc2", "1.14.0-rc2"],
+       "alpha" => ["1.14.0.alpha", "1.14.0-alpha"],
+       "beta" => ["1.14.0.beta", "1.14.0-beta"],
+       "pre" => ["1.14.0.pre", "1.14.0-pre"],
+      )
+  def test_create_go_semver(data)
+    version, expected = data
+    actual = @api_requester.create_go_semver(version)
+    assert_equal expected, actual
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calyptia-monitoring
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc5
+  version: 0.1.0.rc9
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 1.14.0.rc
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +68,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 1.14.0.rc
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -89,7 +89,8 @@ dependencies:
 description: Monitoring Fluentd via Calyptia Cloud
 email:
 - hatake@calyptia.com
-executables: []
+executables:
+- calyptia-config-generator
 extensions: []
 extra_rdoc_files: []
 files:
@@ -98,12 +99,17 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/calyptia-config-generator
 - fluent-plugin-calyptia-monitoring.gemspec
+- lib/fluent/command/config_generator.rb
 - lib/fluent/plugin/calyptia_monitoring_calyptia_api_requester.rb
 - lib/fluent/plugin/calyptia_monitoring_ext.rb
 - lib/fluent/plugin/calyptia_monitoring_machine_id.rb
 - lib/fluent/plugin/in_calyptia_monitoring.rb
+- templates/calyptia-conf.erb
+- test/command/test_config_generator.rb
 - test/helper.rb
+- test/plugin/test_calyptia_monitoring_calyptia_api_requester.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb
 homepage: https://github.com/calyptia/fluent-plugin-calyptia-monitoring
@@ -130,6 +136,8 @@ signing_key:
 specification_version: 4
 summary: Monitoring Fluentd via Calyptia Cloud
 test_files:
+- test/command/test_config_generator.rb
 - test/helper.rb
+- test/plugin/test_calyptia_monitoring_calyptia_api_requester.rb
 - test/plugin/test_calyptia_montoring_machine_id.rb
 - test/plugin/test_in_calyptia_monitoring.rb