fluent-diagtool 1.0.2 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44527fe77df5bcf3b4888b5a697f796460a69e3873561c88b278e318f5c83cd8
-  data.tar.gz: 6052777c8a6930f4dde4bc1fa40bd3a62eba4541253488d419f21f02ab11568a
+  metadata.gz: f767598bf5753ebc1ce4834843a5612d606b0d4d4ac5be26c6538d8d0f7db8fb
+  data.tar.gz: 93cef94976209e28812ab6ca5d466efcdc80c438222477e013d2bc365d8c8cae
 SHA512:
-  metadata.gz: 1d03c3277e64b402db14aaa6f530c02ceb640ff57b53e170fc2633e63d23a53b4b8f604a292b4a958e74035ccb6dac4f02b782ee58e1ea579e916cf3ba050012
-  data.tar.gz: b2d8da326cab4181b391758e9e7fa7cfb8682580726b0921afcc08a6fc8788db818bd325c8a24dcc8ee83975d69d058f306bc8d852b8417a2a4d9f3341e38fd0
+  metadata.gz: 2b45ed8576c82a0d7f772c516127ac66d03d0a2d1051cb34a663e2062c3fb2e27ad8160acba33bf9c7995485d403ae135fc496984a55b05a772d61273ce1aca0
+  data.tar.gz: 2af458d2e1202db45ebe0ddf6e6ab4e75dec2e9f53b995712ce20836a372af86588ddc5e7bb705c947a12c50673c35b8378f55ebb5c0615625c30f789112de95

data/Gemfile.lock

@@ -1,30 +1,30 @@
 PATH
   remote: .
   specs:
-    fluent-diagtool (1.0.1)
+    fluent-diagtool (1.0.4)
       fileutils
       json
 
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.3)
-    fileutils (1.4.1)
-    json (2.3.1)
+    diff-lcs (1.5.1)
+    fileutils (1.7.2)
+    json (2.7.1)
     rake (12.3.3)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
 
 PLATFORMS
   ruby

data/README.md

@@ -21,7 +21,8 @@ The scope of data collection:
 <br>   
 
 ## Prerequisite
-Diagtool has been developed for Fluentd(td-agent) and FluentBit(td-agent-bit) running on Linux OS and Diatool does not work for Windows.
+Diagtool has been developed for Fluentd(td-agent, fluent-package) and FluentBit(td-agent-bit) running on Linux OS, mainly.
+On Windows, it only supports the `installed td-agent-gem list` collection for Fluentd, currently (since v1.0.3).
 Diagtool is written in Ruby and Ruby version should be higher than 2.3 for the installation. 
 The supported Linux OS is described in the following page:  
 https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4
@@ -31,9 +32,9 @@ https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4
 When you are using td-agent, you can install Diagtool easily with "/usr/sbin/td-agent-gem" command.
 ```
 # /usr/sbin/td-agent-gem install fluent-diagtool
-Successfully installed fluent-diagtool-1.0.0
-Parsing documentation for fluent-diagtool-1.0.0
-Installing ri documentation for fluent-diagtool-1.0.0
+Successfully installed fluent-diagtool-1.0.4
+Parsing documentation for fluent-diagtool-1.0.4
+Installing ri documentation for fluent-diagtool-1.0.4
 Done installing documentation for fluent-diagtool after 0 seconds
 1 gem installed
 ```
@@ -42,9 +43,9 @@ When using /usr/sbin/td-agent-gem command, fluent-diagtool is installed under "/
 Otherwise, you can install Diagtool with common gem command. In this case, Ruby version higher than 2.3 might be required to install.
 ```
 # gem install fluent-diagtool
-Successfully installed fluent-diagtool-1.0.0
-Parsing documentation for fluent-diagtool-1.0.0
-Installing ri documentation for fluent-diagtool-1.0.0
+Successfully installed fluent-diagtool-1.0.4
+Parsing documentation for fluent-diagtool-1.0.4
+Installing ri documentation for fluent-diagtool-1.0.4
 Done installing documentation for fluent-diagtool after 0 seconds
 1 gem installed
 ```
@@ -64,7 +65,13 @@ Usage: fluent-diagtool -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {lis
     -c, --conf config_file           provide a full path of td-agent configuration file (Optional : Default=None)
     -l, --log log_file               provide a full path of td-agent log file (Optional : Default=None)
 ```
+
+On Windows, only the `-o, --output DIR` option is supported.
+
 ### Precheck
+
+(Not supported on Windows)
+
 In order to run Diagtool correctly, it is required to ensure that Diagtool can obtain the fundamental information of Fluentd. Basically, Diagtool automatically parses the required information from the running Fluentd processes. The precheck option is useful to confirm if Diagtool certainly collects the information as expected. 
 The following output example shows the case where Diatool properly collects the required information.
 
@@ -162,6 +169,21 @@ Once the pre-check is completed, you are ready to run the tool. The “-o” is
 2020-10-07 21:29:30 +0000: [Diagtool] [INFO] [Collect] Generate tar file /tmp/diagout-20201007212928.tar.gz
 ```
 
+fluent-package (td-agent) on Windows: Fluent Package Command Prompt (Td-agent Command Prompt) with Administrator privilege
+
+```
+$ fluent-diagtool -o /opt
+2023-11-21 12:46:08 +0900: [Diagtool] [INFO] Parsing command options...
+2023-11-21 12:46:08 +0900: [Diagtool] [INFO]    Option : Output directory = /opt
+2023-11-21 12:46:08 +0900: [Diagtool] [INFO] Initializing parameters...
+2023-11-21 12:46:08 +0900: [Diagtool] [INFO] [Collect] Collecting fluent-package gem information...
+2023-11-21 12:46:10 +0900: [Diagtool] [INFO] [Collect] fluent-package gem information is stored in /opt/20231121124608/output/tdgem_list.output
+2023-11-21 12:46:10 +0900: [Diagtool] [INFO] [Collect] fluent-package gem information (bundled by default) is stored in /opt/20231121124608/output/gem_bundled_list.output
+2023-11-21 12:46:10 +0900: [Diagtool] [INFO] [Collect] fluent-package manually installed gem information is stored in /opt/20231121124608/output/gem_local_list.output
+2023-11-21 12:46:10 +0900: [Diagtool] [INFO] [Collect] fluent-package manually installed gems:
+2023-11-21 12:46:10 +0900: [Diagtool] [INFO] [Collect]   * fluent-plugin-forest
+```
+
 #### The "@include" directive in td-agent configuration file
 The "@include" directive is a function to reuse configuration defined in other configuration files. Diagtool reads Fluentd configuration and gathers the files described in "@include" directive as well. The details of "@include" directive are described in followed page:  
 https://docs.fluentd.org/configuration/config-file#6-re-use-your-config-the-include-directive
@@ -203,6 +225,8 @@ The diagtool provides a hash-seed option with '-s'. When hash-seed is specified,
 ```
 
 ## Tested Environment
-- OS : CentOS 8.1 / Ubuntu 20.04
+- OS : CentOS 8.1 / Ubuntu 20.04 / Windows Home 10
 - Fluentd : td-agent version 3/4
-  https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3
+  https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4
+- Fluentd : fluent-package version 5
+  https://docs.fluentd.org/quickstart/fluent-package-v5-vs-td-agent

data/exe/fluent-diagtool

@@ -17,9 +17,6 @@
 #
 
 require 'optparse'
-require 'fluent/diagtool/collectutils'
-require 'fluent/diagtool/maskutils'
-require 'fluent/diagtool/validutils'
 require 'fluent/diagtool/diagutils'
 include Diagtool
 

data/lib/fluent/diagtool/collectutils.rb

@@ -19,9 +19,12 @@ require 'open3'
 require 'logger'
 require 'net/http'
 require 'uri'
+require 'fluent/diagtool/windows/collectutils' if Diagtool.windows?
 
 module Diagtool
   class CollectUtils
+    prepend Windows::PlatformSpecificCollectUtils if Diagtool.windows?
+
     def initialize(conf, log_level)
       @logger = Logger.new(STDOUT, level: log_level, formatter: proc {|severity, datetime, progname, msg|
         "#{datetime}: [Collectutils] [#{severity}] #{msg}\n"
@@ -34,8 +37,9 @@ module Diagtool
       @outdir = conf[:outdir]
       @tdenv = {
         'FLUENT_CONF' => '',
-        'TD_AGENT_LOG_FILE' => ''
-      }			
+        'TD_AGENT_LOG_FILE' => '',
+        'FLUENT_PACKAGE_LOG_FILE' => '',
+      }
       @package_name = conf[:package_name]
       @service_name = conf[:service_name]
 
@@ -153,10 +157,11 @@ module Diagtool
     
     def _find_fluentbit_info()
       ### check if the td-agent-bit is run as daemon
-      stdout, stderr, status = Open3.capture3('systemctl cat td-agent-bit')
+
+      stdout, _stderr, status = Open3.capture3("systemctl cat #{@service_name}")
       if status.success?
         if @precheck == false  # SKip if precheck is true
-          File.open(@outdir+'/td-agent-bit_env.output', 'w') do |f|
+          File.open(@outdir+"/#{@service_name}_env.output", 'w') do |f|
             f.puts(stdout)
           end
         end
@@ -180,8 +185,8 @@ module Diagtool
           end
         end
       else
-        ### check if the td-agent-bit is not run as daemon or run FluentdBit with customized script 
-        stdout, stderr, status = Open3.capture3('ps aux | grep fluent-bit | grep -v ".*\(grep\|diagtool\)"')
+        ### check if the td-agent-bit is not run as daemon or run FluentdBit with customized script
+        stdout, _stderr, status = Open3.capture3('ps aux | grep fluent-bit | grep -v ".*\(grep\|diagtool\)"')
         if status.success?
           i = 0
           stdout.split().each do | line |
@@ -403,16 +408,71 @@ module Diagtool
       return output
     end
 
+    def fluent_package?
+      @package_name == "fluent-package"
+    end
+
+    def match_platform?(platforms_option)
+      if Diagtool.windows?
+        platforms_option == "windows_platforms"
+      else
+        platforms_option == "not_windows_platforms"
+      end
+    end
+
+    def gemfile_path
+      if fluent_package?
+        "/opt/fluent/share/Gemfile"
+      else
+        "/opt/td-agent/share/Gemfile"
+      end
+    end
+
+    def collect_bundled_plugins
+      File.read(gemfile_path).lines(chomp: true).grep(/\Agem "fluent-plugin-/).collect do |line|
+        gem_name = line[/\Agem "(fluent-plugin-.+?)"/, 1]
+        platforms_option = line[/platforms: (\w+)/, 1]
+        next gem_name, platforms_option
+      end.select do |gem_name, platforms_option|
+        platforms_option.nil? || match_platform?(platforms_option)
+      end.collect do |gem_name, platforms_option|
+        gem_name
+      end
+    end
+
+    def collect_manually_installed_gems(gemlist_path)
+      installed_gems = File.read(gemlist_path).lines(chomp: true).collect do |line|
+        line.split.first
+      end
+      bundled_plugins = collect_bundled_plugins
+      bundled_gem_path = File.join(@outdir, "gem_bundled_list.output")
+      File.open(bundled_gem_path, 'w') do |f|
+        f.puts(bundled_plugins.join("\n"))
+      end
+      local_gem_path = File.join(@outdir, "gem_local_list.output")
+      # the folloing gems are not collected as bundled plugins
+      masked_gems = ["fluentd", "fluent-config-regexp-type", "fluent-logger", "fluent-diagtool"]
+      File.open(local_gem_path, 'w') do |f|
+        f.puts((installed_gems - bundled_plugins - masked_gems).join("\n"))
+      end
+      return {:bundled => bundled_gem_path, :local => local_gem_path}
+    end
+
     def collect_tdgems()
       output = @outdir+'/tdgem_list.output'
-      command = if @package_name == "fluent-package"
+      command = if fluent_package?
                   "fluent-gem"
                 else
                   "td-agent-gem"
                 end
-      stdout, stderr, status = Open3.capture3("#{command} list | grep fluent")
+      stdout, stderr, status = Open3.capture3("#{command} list")
+      gems = stdout.each_line(chomp: true).select do |line|
+        line.include?("fluent")
+      end.collect do |line|
+        line.split.first
+      end
       File.open(output, 'w') do |f|
-        f.puts(stdout)
+        f.puts(gems.join("\n"))
       end
       return output
     end

data/lib/fluent/diagtool/diagutils.rb

@@ -14,15 +14,31 @@
 #    limitations under the License.
 #
 
+module Diagtool
+  ON_WINDOWS = /mingw/.match?(RUBY_PLATFORM)
+  def self.windows?
+    ON_WINDOWS
+  end
+end
+
 require 'logger'
 require 'fileutils'
 require 'fluent/diagtool/collectutils'
 require 'fluent/diagtool/maskutils'
 require 'fluent/diagtool/validutils'
+require 'fluent/diagtool/windows/diagutils' if Diagtool.windows?
 include Diagtool
 
 module Diagtool
   class DiagUtils
+    # TODO: Consider making the logic of this class more abstract and
+    # cutting out the unix-specific logic into a separate module as well.
+    # (Currently, very limited features are supported for Windows.
+    # In order to reduce impact on the existing logic for Unix-like.
+    # only Windows-specific logic is separated into the module, for now.
+    # In the future, the implementation of this class should be more abstract.)
+    prepend Windows::PlatformSpecificDiagUtils if Diagtool.windows?
+
     def initialize(params)
       time = Time.new
       @time_format = time.strftime("%Y%m%d%0k%M%0S")
@@ -33,12 +49,22 @@ module Diagtool
 	      "netstat -plan",
 	      "netstat -s",
       ]
-      if fluent_package?
-        @conf[:package_name] = "fluent-package"
-        @conf[:service_name] = "fluentd"
+      if @conf[:type] == "fluentd"
+        if fluent_package?
+          @conf[:package_name] = "fluent-package"
+          @conf[:service_name] = "fluentd"
+        else
+          @conf[:package_name] = "td-agent"
+          @conf[:service_name] = "td-agent"
+        end
       else
-        @conf[:package_name] = "td-agent"
-        @conf[:service_name] = "td-agent"
+        if fluentbit_package?
+          @conf[:package_name] = "fluent-bit"
+          @conf[:service_name] = "fluent-bit"
+        else
+          @conf[:package_name] = "td-agent-bit"
+          @conf[:service_name] = "td-agent-bit"
+        end
       end
     end
     
@@ -123,9 +149,24 @@ module Diagtool
       tdconf = c.collect_tdconf()
       diaglogger_info("[Collect] config file is stored in #{tdconf}")
 
-      diaglogger_info("[Collect] Collecting #{@conf[:package_name]} gem information...")
-      tdgem = c.collect_tdgems()
-      diaglogger_info("[Collect] #{@conf[:package_name]} gem information is stored in #{tdgem}")
+      case @conf[:type]
+      when 'fluentd'
+        diaglogger_info("[Collect] Collecting #{@conf[:package_name]} gem information...")
+        tdgem = c.collect_tdgems()
+        diaglogger_info("[Collect] #{@conf[:package_name]} gem information is stored in #{tdgem}")
+        gem_info = c.collect_manually_installed_gems(tdgem)
+        diaglogger_info("[Collect] #{@conf[:package_name]} gem information (bundled by default) is stored in #{gem_info[:bundled]}")
+        diaglogger_info("[Collect] #{@conf[:package_name]} manually installed gem information is stored in #{gem_info[:local]}")
+        local_gems = File.read(gem_info[:local]).lines(chomp: true)
+        unless local_gems == [""]
+          diaglogger_info("[Collect] #{@conf[:package_name]} manually installed gems:")
+          local_gems.each do |gem|
+            diaglogger_info("[Collect]   * #{gem}")
+          end
+        end
+      when 'fluentbit'
+        # nothing to do!
+      end
 
       diaglogger_info("[Collect] Collecting config file of OS log...")
       oslog = c.collect_oslog()
@@ -324,5 +365,9 @@ module Diagtool
     def fluent_package?
       File.exist?("/etc/fluent/fluentd.conf") || File.exist?("/opt/fluent/bin/fluentd")
     end
+
+    def fluentbit_package?
+      File.exist?("/etc/fluent-bit/fluent-bit.conf") || File.exist?("/opt/fluent-bit/bin/fluent-bit")
+    end
   end
 end

data/lib/fluent/diagtool/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Diagtool
-    VERSION = "1.0.2"
+    VERSION = "1.0.4"
   end
 end

data/lib/fluent/diagtool/windows/collectutils.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Diagtool
+  module Windows
+    module PlatformSpecificCollectUtils
+      def _find_fluentd_info()
+        # Not supported yet.
+      end
+
+      def _find_fluentbit_info()
+        # Not supported yet.
+      end
+
+      def _find_os_info()
+        # Not supported yet.
+        return {}
+      end
+
+      def gemfile_path
+        if fluent_package?
+          File.join(ENV["FLUENT_PACKAGE_TOPDIR"], "share/Gemfile")
+        else
+          File.join(ENV["TD_AGENT_TOPDIR"], "share/Gemfile")
+        end
+      end
+    end
+  end
+end

data/lib/fluent/diagtool/windows/diagutils.rb

@@ -0,0 +1,101 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'logger'
+require 'fileutils'
+require 'fluent/diagtool/collectutils'
+
+module Diagtool
+  module Windows
+    module PlatformSpecificDiagUtils
+      def fluent_package?
+        ENV.key?("FLUENT_PACKAGE_TOPDIR")
+      end
+
+      def run_precheck()
+        raise "[Precheck] Precheck feature is not supported on Windows."
+      end
+
+      def run_diagtool()
+        @conf[:time] = @time_format
+        @conf[:workdir] = @conf[:basedir] + '/' + @time_format
+        @conf[:outdir] = @conf[:workdir] + '/output'
+        FileUtils.mkdir_p(@conf[:workdir])
+        FileUtils.mkdir_p(@conf[:outdir])
+        diaglog = @conf[:workdir] + '/diagtool.output'
+
+        @logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+          "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+        })
+        @logger_file = Logger.new(diaglog, formatter: proc {|severity, datetime, progname, msg|
+          "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+        })
+        diaglogger_info("Parsing command options...")
+        diaglogger_info("   Option : Output directory = #{@conf[:basedir]}")
+
+        loglevel = 'WARN'
+        diaglogger_info("Initializing parameters...")
+        c = CollectUtils.new(@conf, loglevel)
+
+        diaglogger_info("[Collect] Collecting #{@conf[:package_name]} gem information...")
+        tdgem = c.collect_tdgems()
+        diaglogger_info("[Collect] #{@conf[:package_name]} gem information is stored in #{tdgem}")
+
+        gem_info = c.collect_manually_installed_gems(tdgem)
+        diaglogger_info("[Collect] #{@conf[:package_name]} gem information (bundled by default) is stored in #{gem_info[:bundled]}")
+        diaglogger_info("[Collect] #{@conf[:package_name]} manually installed gem information is stored in #{gem_info[:local]}")
+        local_gems = File.read(gem_info[:local]).lines(chomp: true)
+        unless local_gems == [""]
+          diaglogger_info("[Collect] #{@conf[:package_name]} manually installed gems:")
+          local_gems.each do |gem|
+            diaglogger_info("[Collect]   * #{gem}")
+          end
+        end
+      end
+
+      def parse_diagconf(params)
+        options = {
+          :precheck => '', :basedir => '', :type =>'', :mask => '', :words => [], :wfile => '', :seed => '', :tdconf =>'', :tdlog => ''
+        }
+
+        supported_options = [:type, :output]
+
+        unless params[:type] == nil || params[:type] == 'fluentd'
+          raise "fluentd type '-t' only supports 'fluentd' on Windows."
+        end
+        options[:type] = 'fluentd'
+
+        if params[:output] != nil
+          if Dir.exist?(params[:output])
+            options[:basedir] = params[:output]
+          else
+            raise "output directory '#{params[:output]}' does not exist"
+          end
+        else
+          raise "output directory '-o' must be specified"
+        end
+
+        params.keys.each do |option|
+          unless supported_options.include?(option)
+            raise "#{option} is not supported on Windows."
+          end
+        end
+
+        return options
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-diagtool
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.4
 platform: ruby
 authors:
 - kubotat
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-30 00:00:00.000000000 Z
+date: 2024-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fileutils
@@ -66,11 +66,13 @@ files:
 - lib/fluent/diagtool/maskutils.rb
 - lib/fluent/diagtool/validutils.rb
 - lib/fluent/diagtool/version.rb
+- lib/fluent/diagtool/windows/collectutils.rb
+- lib/fluent/diagtool/windows/diagutils.rb
 homepage: https://github.com/fluent/diagtool/tree/dev
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -85,8 +87,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
-signing_key: 
+rubygems_version: 3.4.19
+signing_key:
 specification_version: 4
 summary: Diagnostic Tool for Fluentd
 test_files: []