fluent-diagtool 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cec6d985c53b3d147c6d1eb5418ae15983b1bf2ec1bcd15f6c4c035f20a93cf3
-  data.tar.gz: b58b0565d227ccc1f99d0dc3fbb482e03b1cb5faf38c9a5326fe753f0a222efc
+  metadata.gz: 1afbc81ebf5fe6ab013818ac124776418b3c909bef5ac5a649210499c14ff2c9
+  data.tar.gz: ec4b237dcbe9f065f6645181c5cade9cc118cdfbc376460fbbf3108014d360f0
 SHA512:
-  metadata.gz: c1f5582d4f9835d158f09ee7256771406cf57c8753b6de9170c68d5b4f774ebb7397fce7fe70c8d247eb92bedf9f46a6af6f575b783a4718323b48a2a824b521
-  data.tar.gz: 139b59e1c25cd34e745ba799d5c323c18421d3f5b4b95fbf9c59c63b75291e6663ee9a5cb43cb3cb8ac75ea99a1a495b6c3f9b731960643a93df0fc09e3ddfc1
+  metadata.gz: 71a1b513b1d55e3033ce8cd90d3060a6316c48358f2a374c75f23b43ddb32693dcb0107a96706a1c961445d23b93ada33edfaf7cda37c3ff415482af58c661a1
+  data.tar.gz: 92d4ed8273c892e623487ec0089a606d30bb14798d46c73d7c2469d2c167c8489293776c9750b0a1812edb9d61df0c70c3d2fb1caf75243cd8ed16ee8ae3ac63

data/.gitignore

@@ -0,0 +1,13 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version
+
+test/tmp/

data/AUTHORS

@@ -1 +1 @@
-TOMONORI KUBOTA <tmnr.kubota _at_ gmail.com>
+TOMONORI KUBOTA <tkubota _at_ ctc-america.com>

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    fluent-diagtool (0.1.2)
-      fileutils (~> 1.0.2)
-      json (~> 2.1.0)
+    fluent-diagtool (0.1.7)
+      fileutils
+      json
 
 GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.3)
-    fileutils (1.0.2)
-    json (2.1.0)
+    fileutils (1.4.1)
+    json (2.3.0)
     rake (12.3.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)

data/README.md

@@ -1,7 +1,7 @@
 # Fluentd Diagnostic Tool
 
-The diagtool enable users to automate the date collection which is required for trouble shooting. The data collected by diagtool include the configuration and log files of the td-agent and diagnostic information of operating system such as network and memory status and stats. In some cases, configuration and log files contains the security sensitive information, such as IP addresses and Hostname. The diagtool also provides the functions to generate mask on IP addresses, Hostname(in FQDN style) and user defined keywords described in the collected data.<br> 
-The scope of data collection:<br>
+The diagtool enable users to automate the date collection which is required for trouble shooting. The data collected by diagtool include the configuration and log files of the td-agent and diagnostic information of operating system such as network and memory status and stats. In some cases, configuration and log files contains the security sensitive information, such as IP addresses and Hostname. The diagtool also provides the functions to generate mask on IP addresses, Hostname(in FQDN style) and user defined keywords described in the collected data.   
+The scope of data collection:  
 - TD Agent information
   - configuration files (*)
   - log files (*)
@@ -11,18 +11,20 @@ The scope of data collection:<br>
   - OS log file
   - OS parameters
     - OS and kernel version
-    - time/date information
-    - maximum number of file descriptor(ulimit)
+    - time/date information(ntp -q/chronyc sources)
+    - maximum number of file descriptor(ulimit -n)
     - kernel network parameters(sysctl)
-  - network conectivity status/stats
-  - memory information
-<br>  
-(*) The diagtool automatically gather the path of td-agent configuration files and log files and use them during data collection. 
+  - snapshot of current process(ps)
+  - network conectivity status/stats(netstat -plan/netstat -s)
+  - memory information(/proc/meminfo)
+<br>   
 
 ## Prerequisite
+The diagtool provides support for td-agent based installation running on Linux OS. The td-agent is a stable distribution package of Fluentd.  
+The differences between Fluentd and td-agent are described in followed url:  
+https://www.fluentd.org/faqs
 
-
-## Installation
+## Diagtool Installation
 
 ```
 # gem install fluent-diagtool
@@ -40,13 +42,54 @@ Successfully installed fluent-diagtool-0.1.2
 ```
 # diagtool --help
 Usage: /usr/local/bin/diagtool -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {listfile} -s {hash seed}
+        --precheck                   Run Precheck (Optional)
     -o, --output DIR                 Output directory (Mandatory)
     -m, --mask yes|no                Enable mask function (Optional : Default=no)
     -w, --word-list word1,word2      Provide a list of user-defined words which will to be masked (Optional : Default=None)
-    -f, --word-file listfile         provide a file which describes a List of user-defined words (Optional : Default=None)
+    -f, --word-file list_file        provide a file which describes a List of user-defined words (Optional : Default=None)
     -s, --hash-seed seed             provide a word which will be used when generate the mask (Optional : Default=None)
+    -c, --conf config_file           provide a full path of td-agent configuration file (Optional : Default=None)
+    -l, --log log_file               provide a full path of td-agent log file (Optional : Default=None)
+```
+### Pre-check
+The diagtool automatically extract the path of td-agent configuration and log files from td-agent daemon and use them during data collection if the td-agent is managed as daemon. The precheck options provides the function to confirm if the diagtool could gather the td-agent information as expected. 
+The following command output shows the case when the diagtool successfully gather information from daemon.
+```
+# diagtool --precheck
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck] Check OS parameters...
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck]    operating system = CentOS Linux 8 (Core)
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck]    kernel version = Linux 4.18.0-147.el8.x86_64
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck] Check td-agent parameters...
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck]    td-agent conf path = /etc/td-agent/
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck]    td-agent conf file = td-agent.conf
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck]    td-agent log path = /var/log/td-agent/
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck]    td-agent log = td-agent.log
+2020-05-28 00:39:02 -0400: [Diagtool] [INFO] [Precheck] Precheck completed. You can run diagtool command without -c and -l options
+```
+In some cases, users do not manage td-agent as daemon but use own script to run td-agent with command line options. In that cases, users need to speccify the path of td-agent configuration and log files with -c and -l options respectively.  
+The following example shows the precheck results when the diagtool is not able to extract the path of td-agent configuration and log files.
 ```
-The list of user-defined words can be specified both -e option and -f option.
+# diagtool --precheck
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck] Check OS parameters...
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck]    operating system = CentOS Linux 8 (Core)
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck]    kernel version = Linux 4.18.0-147.5.1.el8_1.x86_64
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck] Check td-agent parameters...
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck]    td-agent conf path =
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck]    td-agent conf file =
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck]    td-agent log path =
+2020-05-28 05:45:14 +0000: [Diagtool] [INFO] [Precheck]    td-agent log =
+2020-05-28 05:45:14 +0000: [Diagtool] [WARN] [Precheck]    can not find td-agent conf path: please run diagtool command with -c /path/to/<td-agent conf file>
+2020-05-28 05:45:14 +0000: [Diagtool] [WARN] [Precheck]    can not find td-agent log path: please run diagtool command with -l /path/to/<td-agent log file>
+```
+
+### Run diagtool
+
+#### The "@include" directive in td-agent configuration file
+The "@include" directive is a function to reuse configuration defined in another configuration files. The diagtool read the td-agent configuration and collect the files described in "@include" directive as well. The details of "@include" directive are described in followed url:  
+https://docs.fluentd.org/configuration/config-file#6-re-use-your-config-the-include-directive
+
+#### User defined words to be masked 
+The user-defined words can be specified both -e option and -f option and the words are merged when both options are selected.
 The format of user-defined words list file specified in -f option should be followed format.
 ```
 # cat word_list_sample
@@ -113,9 +156,9 @@ NOTE: When user specified the keywork, only the exact match words will be masked
 2020-05-12 18:21:22 -0400: [Diagtool] [INFO] [Mask] Export mask log file : ./mask_20200512182119.json
 2020-05-12 18:21:22 -0400: [Diagtool] [INFO] [Collect] Generate tar file /tmp/work1/diagout-20200512182119.tar.gz
 ```
-## Mask Function
-When run diagtool with mask option, the log of mask is also created in 'mask_{timestamp}.json' file. Users are able to confirm how the mask was generated on each files. 
-<br>
+#### Mask Function
+When run diagtool with mask option, the log of mask is also created in 'mask_{timestamp}.json' file. Users are able to confirm how the mask was generated on each files.  
+The diagtool provides hash-seed option with '-s'. When hash-seed is specified, the mask will be generated with original word and hash-seed so that users could use unique mask value.
 #### Mask sample - IP address: IPv4_{md5hash}
 ```
     "Line112-8": {

data/bin/diagtool.rb

@@ -0,0 +1,87 @@
+require 'optparse'
+require 'logger'
+require '../lib/diagutils'
+include Diagtool
+
+logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+  "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+})
+
+output_dir = '../output'
+mask = 'yes'
+exlist= Array.new
+
+opt = OptionParser.new
+opt.banner = "Usage: #{$0} -o OUTPUT_DIR -m {yes | no} -e {word1,[word2...]} -f {listfile}"
+opt.on('-o','--output DIR', String, 'Output directory (Default=./output)') { |o|
+	output_dir = o 
+}
+opt.on('-m','--mask YES|NO', String, 'Enable mask function (Default=True)') { |m| 
+	if m == 'yes' || m == 'no'
+		mask = m
+	else
+		logger.error("Invalid value '#{m}' : -m | --mask should be yes or no")
+		exit!
+	end
+}
+opt.on('-e','--exclude-list LIST', Array, 'Provide a list of exclude words which will to be masked (Default=None)') { |e| exlist += e } 
+opt.on('-f','--exclude-file FILE', String, 'provide a file which describes a List of exclude words (Default=None)') { |f|
+	if File.exist?(f)
+		File.readlines(f).each do  |l|
+			exlist.append(l.gsub(/\n/,''))
+		end
+	else
+		logger.error("No such file or directory")
+		exit!
+	end
+}
+opt.parse(ARGV)
+exlist = exlist.uniq
+
+logger.info("Parsing command options...")
+logger.info("   Option : Output directory = #{output_dir}")
+logger.info("   Option : Mask = #{mask}")
+logger.info("   Option : Exclude list = #{exlist}")
+
+logger.info("Initializing parameters...")
+node1 = Diagutils.new(output_dir,exlist, 'INFO')
+
+logger.info("Collecting log files of td-agent...")
+tdlog = node1.collect_tdlog()
+logger.info("log files of td-agent are stored in #{tdlog}")
+
+logger.info("Collecting config file of td-agent...")
+tdconf = node1.collect_tdconf()
+logger.info("config file is stored in #{tdconf}")
+
+logger.info("Collecting systctl information...")
+sysctl = node1.collect_sysctl()
+logger.info("sysctl information is stored in #{sysctl}")
+
+logger.info("Collecting date/time information...")
+ntp = node1.collect_ntp()
+logger.info("date/time information is stored in #{ntp}")
+
+logger.info("Collecting ulimit information...")
+ulimit = node1.collect_ulimit()
+logger.info("ulimit information is stored in #{ulimit}")
+
+if mask == 'yes'
+	logger.info("Masking td-agent config file : #{tdconf}...")
+	node1.mask_tdconf(tdconf)
+	tdlog.each do | file |
+		logger.info("Masking td-agent log file : #{file}...")
+      		filename = file.split("/")[-1]
+		if filename.include?(".gz")
+               		node1.mask_tdlog_gz(file)
+       		elsif
+               		node1.mask_tdlog(file)
+       		end
+	end
+end
+
+tar_file = node1.compress_output()
+logger.info("Generate tar file #{tar_file}")
+
+
+

data/exclude_list01

@@ -0,0 +1,2 @@
+centos8101
+centos8102

data/exe/diagtool

@@ -26,13 +26,21 @@ include Diagtool
 params = {}
 OptionParser.new do |opt|
   opt.banner = "Usage: #{$0} -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {listfile} -s {hash seed}"
+  opt.on('--precheck', 'Run Precheck (Optional)')
   opt.on('-o','--output DIR', String, 'Output directory (Mandatory)')
   opt.on('-m','--mask yes|no', String, 'Enable mask function (Optional : Default=no)')
   opt.on('-w','--word-list word1,word2', Array, 'Provide a list of user-defined words which will to be masked (Optional : Default=None)')
-  opt.on('-f','--word-file listfile', String, 'provide a file which describes a List of user-defined words (Optional : Default=None)')
+  opt.on('-f','--word-file list_file', String, 'provide a file which describes a List of user-defined words (Optional : Default=None)')
   opt.on('-s','--hash-seed seed', String, 'provide a word which will be used when generate the mask (Optional : Default=None)')
+  opt.on('-c','--conf config_file', String, 'provide a full path of td-agent configuration file (Optional : Default=None)')
+  opt.on('-l','--log log_file', String, 'provide a full path of td-agent log file (Optional : Default=None)')
 end.parse!(into: params)
+
 diag = DiagUtils.new(params)
-diag.diagtool()
+if params[:precheck]
+  diag.run_precheck()
+else
+  diag.run_diagtool()
+end
 
 

data/fluent-diagtool.gemspec

@@ -19,6 +19,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency("fileutils", ["~> 1.0"])
-  spec.add_runtime_dependency("json", ["~> 2.1"])
+  spec.add_runtime_dependency("fileutils")
+  spec.add_runtime_dependency("json")
 end

data/lib/fluent/diagtool/collectutils.rb

@@ -22,22 +22,41 @@ module Diagtool
   class CollectUtils
     def initialize(conf, log_level)
       @logger = Logger.new(STDOUT, level: log_level, formatter: proc {|severity, datetime, progname, msg|
-        "#{datetime}: [Diagutils] [#{severity}] #{msg}\n"
+        "#{datetime}: [Collectutils] [#{severity}] #{msg}\n"
       })
+      @precheck = conf[:precheck]
       @time_format = conf[:time]
       @basedir = conf[:basedir]
       @workdir = conf[:workdir]
       @outdir = conf[:outdir]			
-
-      @tdenv = get_tdenv()
-      @tdconf = @tdenv['FLUENT_CONF'].split('/')[-1]
-      @tdconf_path = @tdenv['FLUENT_CONF'].gsub(@tdconf,'')
-      @tdlog =  @tdenv['TD_AGENT_LOG_FILE'].split('/')[-1]
-      @tdlog_path = @tdenv['TD_AGENT_LOG_FILE'].gsub(@tdlog,'')
-
-      @osenv = get_osenv()
+       
+      @tdenv = gen_tdenv()
+      if not conf[:tdconf].empty?
+	@tdconf = conf[:tdconf].split('/')[-1]
+        @tdconf_path = conf[:tdconf].gsub(@tdconf,'')
+      elsif
+	if not @tdenv['FLUENT_CONF'].empty?
+      	  @tdconf = @tdenv['FLUENT_CONF'].split('/')[-1]
+      	  @tdconf_path = @tdenv['FLUENT_CONF'].gsub(@tdconf,'')
+	else
+	  raise "The path of td-agent configuration file need to be specified."  if conf[:precheck] == false
+	end
+      end
+      if not conf[:tdlog].empty?
+        @tdlog = conf[:tdlog].split('/')[-1]
+        @tdlog_path = conf[:tdlog].gsub(@tdlog,'')
+      elsif
+        if not @tdenv['TD_AGENT_LOG_FILE'].empty?
+          @tdlog =  @tdenv['TD_AGENT_LOG_FILE'].split('/')[-1]
+          @tdlog_path = @tdenv['TD_AGENT_LOG_FILE'].gsub(@tdlog,'')
+        else
+          raise "The path of td-agent log file need to be specified." if conf[:precheck] == false
+	end
+      end 
+      @osenv = gen_osenv()
       @oslog_path = '/var/log/'
       @oslog = 'messages'
+      @syslog = 'syslog'
       @sysctl_path = '/etc/'
       @sysctl = 'sysctl.conf'        
 
@@ -50,29 +69,61 @@ module Diagtool
       @logger.info("    td-agent log = #{@tdlog}")
     end
     
-    def get_osenv()
+    def gen_osenv()
       stdout, stderr, status = Open3.capture3('hostnamectl')
       os_dict = {}
       stdout.each_line { |l|
         s = l.split(":")
         os_dict[s[0].chomp.strip] = s[1].chomp.strip
       }
-      File.open(@workdir+'/os_env.output', 'w') do |f|
-        f.puts(stdout)
+      if @precheck == false  # SKip if precheck is true
+        File.open(@outdir+'/os_env.output', 'w') do |f|
+          f.puts(stdout)
+        end
       end
       return os_dict
     end
     
-    def get_tdenv()
+    def gen_tdenv()
       stdout, stderr, status = Open3.capture3('systemctl cat td-agent')
       env_dict = {}
-        File.open(@workdir+'/td-agent_env.output', 'w') do |f|
-          f.puts(stdout)
-        end
+      if status.success?
+	if @precheck == false  # SKip if precheck is true
+          File.open(@outdir+'/td-agent_env.output', 'w') do |f|
+            f.puts(stdout)
+          end
+	end  
         stdout.split().each do | l |
-        if l.include?('Environment')
-          env_dict[l.split('=')[1]] = l.split('=')[2]
-        end
+          if l.include?('Environment')
+            env_dict[l.split('=')[1]] = l.split('=')[2]
+          end
+      	end
+      else
+        exe = 'fluentd'
+        stdout, stderr, status = Open3.capture3("ps aux | grep #{exe} | grep -v grep")
+        line = stdout.split(/\n/)
+	log_path = ''
+        conf_path = ''
+        line.each { |l|
+          cmd = l.split.drop(10)
+          i = 0
+          log_pos = 0
+          conf_pos = 0
+          if cmd[-1] != '--under-supervisor'
+            cmd.each { |c|
+              if c.include?("--log") || c.include?("-l")
+                log_pos = i + 1
+                log_path = cmd[log_pos]
+              elsif c.include?("--conf") || c.include?("-c")
+                conf_pos = i + 1
+                conf_path = cmd[conf_pos]
+              end
+              i+=1
+            }
+	  end
+	}
+        env_dict['FLUENT_CONF'] = conf_path
+        env_dict['TD_AGENT_LOG_FILE'] = log_path
       end
       return env_dict
     end
@@ -90,86 +141,83 @@ module Diagtool
     end
     
     def collect_tdconf()
-      FileUtils.mkdir_p(@workdir+@tdconf_path)
-      FileUtils.cp(@tdconf_path+@tdconf, @workdir+@tdconf_path)
-      return @workdir+@tdconf_path+@tdconf
+      target_dir = @workdir+@tdconf_path
+      FileUtils.mkdir_p(target_dir)
+      FileUtils.cp(@tdconf_path+@tdconf, target_dir)
+      conf = @workdir+@tdconf_path+@tdconf
+      conf_list = []
+      conf_list.push target_dir + @tdconf
+      File.readlines(conf).each { |line|
+      if line.include? '@include'
+        f = line.split()[1]
+        if f.start_with?(/\//)  # /tmp/work1/b.conf
+          if f.include?('*')
+            Dir.glob(f).each { |ff|
+              conf_inc = target_dir + ff.gsub(/\//,'__')
+              FileUtils.cp(ff, conf_inc)
+              conf_list.push conf_inc
+             }
+	  else 
+	    conf_inc = target_dir+f.gsub(/\//,'__')
+            FileUtils.cp(f, conf_inc)
+            conf_list.push  conf_inc
+	  end
+        else
+	  f = f.gsub('./','') if f.include?('./')
+          if f.include?('*')
+            Dir.glob(@tdconf_path+f).each{ |ff|
+              conf_inc = target_dir + ff.gsub(@tdconf_path,'').gsub(/\//,'__')
+              FileUtils.cp(ff, conf_inc)
+              conf_list.push conf_inc
+            }
+	  else
+            conf_inc = target_dir+f.gsub(/\//,'__')
+            FileUtils.cp(@tdconf_path+f, conf_inc)
+            conf_list.push  conf_inc
+	  end
+        end
+      end
+     }
+     return conf_list
     end
-    
+
     def collect_tdlog()
-      FileUtils.mkdir_p(@workdir+@tdlog_path)
-      FileUtils.cp_r(@tdlog_path, @workdir+@oslog_path)
-      return Dir.glob(@workdir+@tdlog_path+@tdlog+'*')
-    end
-    
-    def collect_sysctl()
-      FileUtils.mkdir_p(@workdir+@sysctl_path)
-      FileUtils.cp(@sysctl_path+@sysctl, @workdir+@sysctl_path)
-      return @workdir+@sysctl_path+@sysctl
+      target_dir = @workdir+@tdlog_path
+      FileUtils.mkdir_p(target_dir)
+      Dir.glob(@tdlog_path+@tdlog+'*').each{ |f| 
+        FileUtils.cp(f, target_dir)
+      }
+      return Dir.glob(target_dir+@tdlog+'*')
     end
     
     def collect_oslog()
-      FileUtils.mkdir_p(@workdir+@oslog_path)
-      FileUtils.cp(@oslog_path+@oslog, @workdir+@oslog_path)
-      return @workdir+@oslog_path+@oslog
-    end
-    
-    def collect_ulimit()
-      output = @outdir+'/ulimit_n.output'
-      stdout, stderr, status = Open3.capture3("ulimit -n")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
+      target_dir = @workdir+@oslog_path
+      FileUtils.mkdir_p(target_dir)
+      if File.exist? @oslog_path+@oslog
+      	FileUtils.cp(@oslog_path+@oslog, target_dir)
+      	return target_dir+@oslog
+      elsif File.exist? @oslog_path+@syslog
+        FileUtils.cp(@oslog_path+@syslog, target_dir)
+        return target_dir+@syslog
+      else
+	@logger.warn("Can not find OS log file in #{oslog} or #{syslog}")
       end
-      return output
-    end
-   
-    def collect_ps_eo()
-      output = @outdir+'/ps_eo.output'
-      stdout, stderr, status = Open3.capture3("ps -eo pid,ppid,stime,time,%mem,%cpu,cmd")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
     end
 
-    def collect_meminfo()
-      output = @outdir+'/meminfo.output'
-      stdout, stderr, status = Open3.capture3("cat /proc/meminfo")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-    
-    def collect_netstat_plan()
-      output = @outdir+'/netstat_plan.output'
-      stdout, stderr, status = Open3.capture3("netstat -plan")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-    
-    def collect_netstat_s()
-      output = @outdir+'/netstat_s.output'
-      stdout, stderr, status = Open3.capture3("netstat -s")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-    
-    def collect_ntp(command)
-      output = @outdir+'/ntp_info.output'
-      stdout_date, stderr_date, status_date = Open3.capture3("date")
-      stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("chronyc sources") if command == "chrony"
-      stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("ntpq -p") if command == "ntp"
-      File.open(output, 'w') do |f|
-        f.puts(stdout_date)
-        f.puts(stdout_ntp)
+    def collect_cmd_output(cmd)
+      cmd_name = cmd.gsub(/\s/,'_').gsub(/\//,'-').gsub(',','_')
+      output = @outdir+'/'+cmd_name+'.txt'
+      stdout, stderr, status = Open3.capture3(cmd)
+      if status.success?
+	File.open(output, 'w') do |f|
+          f.puts(stdout)
+        end
+      else
+        @logger.warn("Command #{cmd} failed due to the following message -  #{stderr.chomp}")
       end
       return output
     end
-    
+
     def collect_tdgems()
       output = @outdir+'/tdgem_list.output'
       stdout, stderr, status = Open3.capture3("td-agent-gem list | grep fluent")

data/lib/fluent/diagtool/diagutils.rb

@@ -27,14 +27,48 @@ module Diagtool
       time = Time.new
       @time_format = time.strftime("%Y%m%d%0k%M%0S")
       @conf = parse_diagconf(params)
+      @cmd_list = [
+      	"ps -eo pid,ppid,stime,time,%mem,%cpu,cmd",
+	"cat /proc/meminfo",
+	"netstat -plan",
+	"netstat -s",
+      ]
+    end
+    
+    def run_precheck()
+      prechecklog = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+        "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+      })
+      loglevel = 'WARN'
+      c = CollectUtils.new(@conf, loglevel)
+      c_env = c.export_env()
+      prechecklog.info("[Precheck] Check OS parameters...")
+      prechecklog.info("[Precheck]    operating system = #{c_env[:os]}")
+      prechecklog.info("[Precheck]    kernel version = #{c_env[:kernel]}")
+      prechecklog.info("[Precheck] Check td-agent parameters...")
+      prechecklog.info("[Precheck]    td-agent conf path = #{c_env[:tdconf_path]}")
+      prechecklog.info("[Precheck]    td-agent conf file = #{c_env[:tdconf]}")
+      prechecklog.info("[Precheck]    td-agent log path = #{c_env[:tdlog_path]}")
+      prechecklog.info("[Precheck]    td-agent log = #{c_env[:tdlog]}")
+      if c_env[:tdconf_path] == nil || c_env[:tdconf] == nil
+	prechecklog.warn("[Precheck]    can not find td-agent conf path: please run diagtool command with -c /path/to/<td-agent conf file>")
+      end
+      if c_env[:tdlog_path] == nil || c_env[:tdlog] == nil
+        prechecklog.warn("[Precheck]    can not find td-agent log path: please run diagtool command with -l /path/to/<td-agent log file>")
+      end
+      if c_env[:tdconf_path] != nil && c_env[:tdconf] != nil && c_env[:tdlog_path] != nil && c_env[:tdlog] != nil
+	 prechecklog.info("[Precheck] Precheck completed. You can run diagtool command without -c and -l options")
+      end
+    end
+
+    def run_diagtool()
       @conf[:time] = @time_format
       @conf[:workdir] = @conf[:basedir] + '/' + @time_format
       @conf[:outdir] = @conf[:workdir] + '/output'
-      
       FileUtils.mkdir_p(@conf[:workdir])
       FileUtils.mkdir_p(@conf[:outdir])
-      
       diaglog = @conf[:workdir] + '/diagtool.output'
+
       @masklog = './mask_' + @time_format + '.json'
       @logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
         "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
@@ -47,9 +81,7 @@ module Diagtool
       diaglogger_info("   Option : Mask = #{@conf[:mask]}")
       diaglogger_info("   Option : Word list = #{@conf[:words]}")
       diaglogger_info("   Option : Hash Seed = #{@conf[:seed]}")
-    end
-    
-    def diagtool()
+
       loglevel = 'WARN'
       diaglogger_info("Initializing parameters...")
       c = CollectUtils.new(@conf, loglevel)
@@ -84,39 +116,35 @@ module Diagtool
       end
       diaglogger_info("[Collect] config file is stored in #{oslog}")
 
-      diaglogger_info("[Collect] Collecting process information...")
-      meminfo = c.collect_ps_eo()
-      diaglogger_info("[Collect] process informationis stored in #{meminfo}")
-
-      diaglogger_info("[Collect] Collecting OS memory information...")
-      meminfo = c.collect_meminfo()
-      diaglogger_info("[Collect] OS memory information is stored in #{meminfo}")
-
       diaglogger_info("[Collect] Collecting date/time information...")
       if system('which chronyc > /dev/null 2>&1')
-        ntp = c.collect_ntp(command="chrony")
+        ntp = c.collect_cmd_output(command="chronyc sources")
+	diaglogger_info("[Collect] date/time information is stored in #{ntp}")
       elsif system('which ntpq > /dev/null 2>&1')
-        ntp = c.collect_ntp(command="ntp")
+        ntp = c.collect_ntp(command="ntpq -p")
+	diaglogger_info("[Collect] date/time information is stored in #{ntp}")
       else
         diaglogger_warn("[Collect] chrony/ntp does not exist. skip collectig date/time information")
       end
-      diaglogger_info("[Collect] date/time information is stored in #{ntp}")
-			
-      diaglogger_info("[Collect] Collecting netstat information...")
-      if system('which netstat > /dev/null 2>&1')
-        netstat_n = c.collect_netstat_plan()
-        netstat_s = c.collect_netstat_s()
-        if @conf[:mask] == 'yes'
-          diaglogger_info("[Mask] Masking netstat file : #{netstat_n}...")
-          netstat_n = m.mask_tdlog(netstat_n, clean = true)
-        end
-        diaglogger_info("[Collect] netstat information is stored in #{netstat_n} and #{netstat_s}")		
-      else
-        diaglogger_warn("[Collect] netstat does not exist. skip collectig netstat")
-      end
       
+      ###
+      #  Correct OS information
+      ###
+      @cmd_list.each { |cmd|
+	diaglogger_info("[Collect] Collecting command output : command = #{cmd}")
+	out = c.collect_cmd_output(cmd)
+	if @conf[:mask] == 'yes'
+          diaglogger_info("[Mask] Masking netstat file : #{out}...")
+          out = m.mask_tdlog(out, clean = true)
+        end
+	diaglogger_info("[Collect] Collecting command output #{cmd.split[0]} stored in #{out}")
+      }
+			
+      ###
+      #  Correct information to be validated
+      ###
       diaglogger_info("[Collect] Collecting systctl information...")
-      sysctl = c.collect_sysctl()
+      sysctl = c.collect_cmd_output("sysctl -a")
       diaglogger_info("[Collect] sysctl information is stored in #{sysctl}")
 			
       diaglogger_info("[Valid] Validating systctl information...")
@@ -131,7 +159,7 @@ module Diagtool
       end
 
       diaglogger_info("[Collect] Collecting ulimit information...")
-      ulimit = c.collect_ulimit()
+      ulimit = c.collect_cmd_output(cmd="sh -c 'ulimit -n'")
       diaglogger_info("[Collect] ulimit information is stored in #{ulimit}")
 
       diaglogger_info("[Valid] Validating ulimit information...")
@@ -143,9 +171,11 @@ module Diagtool
       end
 
       if @conf[:mask] == 'yes'
-        diaglogger_info("[Mask] Masking td-agent config file : #{tdconf}...")
-        m.mask_tdlog(tdconf, clean = true)
-        tdlog.each do | file |
+	tdconf.each { | file |
+	  diaglogger_info("[Mask] Masking td-agent config file : #{file}...")
+	  m.mask_tdlog(file, clean = true)
+	}
+        tdlog.each { | file |
           diaglogger_info("[Mask] Masking td-agent log file : #{file}...")
           filename = file.split("/")[-1]
           if filename.include?(".gz")
@@ -153,30 +183,37 @@ module Diagtool
           elsif
             m.mask_tdlog(file, clean = true)
           end
-        end
+	}
       end
       
       if @conf[:mask] == 'yes'
         diaglogger_info("[Mask] Export mask log file : #{@masklog}")
         m.export_masklog(@masklog)
       end
-      
+
       tar_file = c.compress_output()
       diaglogger_info("[Collect] Generate tar file #{tar_file}")
     end
 
     def parse_diagconf(params)
       options = {
-        :basedir => '', :mask => '', :words => [], :wfile => '', :seed => ''
+        :precheck => '', :basedir => '', :mask => '', :words => [], :wfile => '', :seed => '', :tdconf =>'', :tdlog => ''
       }
-      if params[:output] != nil
-        if Dir.exist?(params[:output])
-          options[:basedir] = params[:output]
+      if params[:precheck]
+        options[:precheck] = params[:precheck]
+      else
+        options[:precheck] = false
+      end
+      if options[:precheck] == false
+        if params[:output] != nil
+          if Dir.exist?(params[:output])
+            options[:basedir] = params[:output]
+          else
+            raise "output directory '#{basedir}' does not exist"
+          end
         else
-          raise "output directory '#{basedir}' does not exist"
+          raise "output directory '-o' must be specified"
         end
-      else
-        raise "output directory '-o' must be specified"
       end
       if params[:mask] == nil
         options[:mask] = 'no'
@@ -200,6 +237,25 @@ module Diagtool
       end
       options[:words] = options[:words].uniq 
       options[:seed] = params[:"hash-seed"] if params[:"hash-seed"] != nil
+      
+      if params[:conf] != nil
+        f = params[:conf]
+        if File.exist?(f)
+	  options[:tdconf] = params[:conf]
+        else
+          raise "#{params[:conf]} : No such file or directory"
+        end
+      end
+
+      if params[:log] != nil
+        f = params[:log]
+        if File.exist?(f)
+          options[:tdlog] = params[:log]
+        else
+          raise "#{params[:log]} : No such file or directory"
+        end
+      end
+
       return options	
     end
     

data/lib/fluent/diagtool/validutils.rb

@@ -33,7 +33,8 @@ module Diagtool
         :net_ipv4_tcp_max_syn_backlog => "8096",
         :net_ipv4_tcp_slow_start_after_idle => "0",
         :net_ipv4_tcp_tw_reuse => "1",
-        :net_ipv4_ip_local_port_range => ["10240", "65535"]}
+        :net_ipv4_ip_local_port_range => ["10240", "65535"]
+      }
       @logger.debug("Initialize Validation Utils:")
       @logger.debug("    Default ulimit: #{@def_ulimit}")
       @logger.debug("    Default sysctl: #{@def_sysctl}")
@@ -57,7 +58,7 @@ module Diagtool
       v = Hash.new { |i,j| i[j] = Hash.new(&h.default_proc) }
       @logger.info("Loading sysctl file: #{sysctl_file}")
       File.readlines(sysctl_file).each{ |line|
-        if line.include?("net")
+        if line.include? "net"
           line_net = line.chomp.gsub(".","_").split("=")
           key = line_net[0].strip.to_sym
           if line_net[1].strip! =~ /\s/
@@ -66,17 +67,19 @@ module Diagtool
             value= line_net[1]
           end
           h[key] = value
-          if @def_sysctl[key] == value
-            @logger.info("#{key} => #{value} is correct")
-            v[key]['value'] = value
-            v[key]['recommend'] = @def_sysctl[key]	
-            v[key]['result'] = "correct"
-          else
-            @logger.warn("#{key} => #{value} is incorrect, should be #{@def_sysctl[key]}")
-            v[key]['value'] = value
-            v[key]['recommend'] = @def_sysctl[key]
-            v[key]['result'] = "incorrect"
-          end
+	  if @def_sysctl.key? key
+            if @def_sysctl[key] == value
+              @logger.info("#{key} => #{value} is correct")
+              v[key]['value'] = value
+              v[key]['recommend'] = @def_sysctl[key]	
+              v[key]['result'] = "correct"
+            else
+              @logger.warn("#{key} => #{value} is incorrect, should be #{@def_sysctl[key]}")
+              v[key]['value'] = value
+              v[key]['recommend'] = @def_sysctl[key]
+              v[key]['result'] = "incorrect"
+            end
+	  end
         end
       }
       if h == @sysctl

data/lib/fluent/diagtool/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Diagtool
-    VERSION = "0.1.3"
+    VERSION = "0.1.8"
   end
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: fluent-diagtool
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - kubotat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2020-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fileutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '0'
 description: Bringing productivity of trouble shooting to the next level  by automating
   collection of Fluentd configurations, settings and OS parameters as well as masking
   sensitive information in logs and configurations.
@@ -48,6 +48,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - AUTHORS
 - Gemfile
 - Gemfile.lock
@@ -55,8 +56,10 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/diagtool.rb
 - bin/setup
 - bin/word_list_sample
+- exclude_list01
 - exe/diagtool
 - fluent-diagtool.gemspec
 - lib/fluent/diagtool/collectutils.rb