fluent-diagtool 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08fde300371aa506f74a6b700b18840d9ccfa88059a5b2ddff78256e839c37be'
-  data.tar.gz: 890483b9fa25f5ee44454d0c8e527ecd7dab2f60e676b283fc64d433c48ac6eb
+  metadata.gz: 1afbc81ebf5fe6ab013818ac124776418b3c909bef5ac5a649210499c14ff2c9
+  data.tar.gz: ec4b237dcbe9f065f6645181c5cade9cc118cdfbc376460fbbf3108014d360f0
 SHA512:
-  metadata.gz: 0e4563b5db53d26fe79dc2a1d7f4c425f63a28bc4ebba62c9005796a258d69b92f1ecac2b8e22f952ff9ea672f5e0e6ff10aeaab49b52773ab0cf295b21a13a9
-  data.tar.gz: 5e26af31e777513a58067262e6ea89cad0d7c07f29d3e1e5ceca816162dc089325502f6b5c4a68470be548340c0e9b312e75c0490b1e8346a05526c95b79cd72
+  metadata.gz: 71a1b513b1d55e3033ce8cd90d3060a6316c48358f2a374c75f23b43ddb32693dcb0107a96706a1c961445d23b93ada33edfaf7cda37c3ff415482af58c661a1
+  data.tar.gz: 92d4ed8273c892e623487ec0089a606d30bb14798d46c73d7c2469d2c167c8489293776c9750b0a1812edb9d61df0c70c3d2fb1caf75243cd8ed16ee8ae3ac63

data/bin/diagtool.rb

@@ -0,0 +1,87 @@
+require 'optparse'
+require 'logger'
+require '../lib/diagutils'
+include Diagtool
+
+logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+  "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+})
+
+output_dir = '../output'
+mask = 'yes'
+exlist= Array.new
+
+opt = OptionParser.new
+opt.banner = "Usage: #{$0} -o OUTPUT_DIR -m {yes | no} -e {word1,[word2...]} -f {listfile}"
+opt.on('-o','--output DIR', String, 'Output directory (Default=./output)') { |o|
+	output_dir = o 
+}
+opt.on('-m','--mask YES|NO', String, 'Enable mask function (Default=True)') { |m| 
+	if m == 'yes' || m == 'no'
+		mask = m
+	else
+		logger.error("Invalid value '#{m}' : -m | --mask should be yes or no")
+		exit!
+	end
+}
+opt.on('-e','--exclude-list LIST', Array, 'Provide a list of exclude words which will to be masked (Default=None)') { |e| exlist += e } 
+opt.on('-f','--exclude-file FILE', String, 'provide a file which describes a List of exclude words (Default=None)') { |f|
+	if File.exist?(f)
+		File.readlines(f).each do  |l|
+			exlist.append(l.gsub(/\n/,''))
+		end
+	else
+		logger.error("No such file or directory")
+		exit!
+	end
+}
+opt.parse(ARGV)
+exlist = exlist.uniq
+
+logger.info("Parsing command options...")
+logger.info("   Option : Output directory = #{output_dir}")
+logger.info("   Option : Mask = #{mask}")
+logger.info("   Option : Exclude list = #{exlist}")
+
+logger.info("Initializing parameters...")
+node1 = Diagutils.new(output_dir,exlist, 'INFO')
+
+logger.info("Collecting log files of td-agent...")
+tdlog = node1.collect_tdlog()
+logger.info("log files of td-agent are stored in #{tdlog}")
+
+logger.info("Collecting config file of td-agent...")
+tdconf = node1.collect_tdconf()
+logger.info("config file is stored in #{tdconf}")
+
+logger.info("Collecting systctl information...")
+sysctl = node1.collect_sysctl()
+logger.info("sysctl information is stored in #{sysctl}")
+
+logger.info("Collecting date/time information...")
+ntp = node1.collect_ntp()
+logger.info("date/time information is stored in #{ntp}")
+
+logger.info("Collecting ulimit information...")
+ulimit = node1.collect_ulimit()
+logger.info("ulimit information is stored in #{ulimit}")
+
+if mask == 'yes'
+	logger.info("Masking td-agent config file : #{tdconf}...")
+	node1.mask_tdconf(tdconf)
+	tdlog.each do | file |
+		logger.info("Masking td-agent log file : #{file}...")
+      		filename = file.split("/")[-1]
+		if filename.include?(".gz")
+               		node1.mask_tdlog_gz(file)
+       		elsif
+               		node1.mask_tdlog(file)
+       		end
+	end
+end
+
+tar_file = node1.compress_output()
+logger.info("Generate tar file #{tar_file}")
+
+
+

data/exclude_list01

@@ -0,0 +1,2 @@
+centos8101
+centos8102

data/lib/fluent/diagtool/collectutils.rb

@@ -22,7 +22,7 @@ module Diagtool
   class CollectUtils
     def initialize(conf, log_level)
       @logger = Logger.new(STDOUT, level: log_level, formatter: proc {|severity, datetime, progname, msg|
-        "#{datetime}: [Diagutils] [#{severity}] #{msg}\n"
+        "#{datetime}: [Collectutils] [#{severity}] #{msg}\n"
       })
       @precheck = conf[:precheck]
       @time_format = conf[:time]
@@ -56,6 +56,7 @@ module Diagtool
       @osenv = gen_osenv()
       @oslog_path = '/var/log/'
       @oslog = 'messages'
+      @syslog = 'syslog'
       @sysctl_path = '/etc/'
       @sysctl = 'sysctl.conf'        
 
@@ -189,77 +190,34 @@ module Diagtool
       return Dir.glob(target_dir+@tdlog+'*')
     end
     
-    def collect_sysctl()
-      target_dir = @workdir+@sysctl_path
-      FileUtils.mkdir_p(target_dir)
-      FileUtils.cp(@sysctl_path+@sysctl, target_dir)
-      return target_dir+@sysctl
-    end
-    
     def collect_oslog()
       target_dir = @workdir+@oslog_path
       FileUtils.mkdir_p(target_dir)
-      FileUtils.cp(@oslog_path+@oslog, target_dir)
-      return target_dir+@oslog
-    end
-    
-    def collect_ulimit()
-      output = @outdir+'/ulimit_n.output'
-      stdout, stderr, status = Open3.capture3("sh -c 'ulimit -n'")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-   
-    def collect_ps_eo()
-      output = @outdir+'/ps_eo.output'
-      stdout, stderr, status = Open3.capture3("ps -eo pid,ppid,stime,time,%mem,%cpu,cmd")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
+      if File.exist? @oslog_path+@oslog
+      	FileUtils.cp(@oslog_path+@oslog, target_dir)
+      	return target_dir+@oslog
+      elsif File.exist? @oslog_path+@syslog
+        FileUtils.cp(@oslog_path+@syslog, target_dir)
+        return target_dir+@syslog
+      else
+	@logger.warn("Can not find OS log file in #{oslog} or #{syslog}")
       end
-      return output
     end
 
-    def collect_meminfo()
-      output = @outdir+'/meminfo.output'
-      stdout, stderr, status = Open3.capture3("cat /proc/meminfo")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-    
-    def collect_netstat_plan()
-      output = @outdir+'/netstat_plan.output'
-      stdout, stderr, status = Open3.capture3("netstat -plan")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-    
-    def collect_netstat_s()
-      output = @outdir+'/netstat_s.output'
-      stdout, stderr, status = Open3.capture3("netstat -s")
-      File.open(output, 'w') do |f|
-        f.puts(stdout)
-      end
-      return output
-    end
-    
-    def collect_ntp(command)
-      output = @outdir+'/ntp_info.output'
-      stdout_date, stderr_date, status_date = Open3.capture3("date")
-      stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("chronyc sources") if command == "chrony"
-      stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("ntpq -p") if command == "ntp"
-      File.open(output, 'w') do |f|
-        f.puts(stdout_date)
-        f.puts(stdout_ntp)
+    def collect_cmd_output(cmd)
+      cmd_name = cmd.gsub(/\s/,'_').gsub(/\//,'-').gsub(',','_')
+      output = @outdir+'/'+cmd_name+'.txt'
+      stdout, stderr, status = Open3.capture3(cmd)
+      if status.success?
+	File.open(output, 'w') do |f|
+          f.puts(stdout)
+        end
+      else
+        @logger.warn("Command #{cmd} failed due to the following message -  #{stderr.chomp}")
       end
       return output
     end
-    
+
     def collect_tdgems()
       output = @outdir+'/tdgem_list.output'
       stdout, stderr, status = Open3.capture3("td-agent-gem list | grep fluent")

data/lib/fluent/diagtool/diagutils.rb

@@ -27,26 +27,12 @@ module Diagtool
       time = Time.new
       @time_format = time.strftime("%Y%m%d%0k%M%0S")
       @conf = parse_diagconf(params)
-      #@conf[:time] = @time_format
-      #@conf[:workdir] = @conf[:basedir] + '/' + @time_format
-      #@conf[:outdir] = @conf[:workdir] + '/output'
-
-      #FileUtils.mkdir_p(@conf[:workdir])
-      #FileUtils.mkdir_p(@conf[:outdir])
-      
-      #diaglog = @conf[:workdir] + '/diagtool.output'
-      #@masklog = './mask_' + @time_format + '.json'
-      #@logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
-      #  "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
-      #})
-      #@logger_file = Logger.new(diaglog, formatter: proc {|severity, datetime, progname, msg|
-      #  "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
-      #})
-      #diaglogger_info("Parsing command options...")
-      #diaglogger_info("   Option : Output directory = #{@conf[:basedir]}")
-      #diaglogger_info("   Option : Mask = #{@conf[:mask]}")
-      #diaglogger_info("   Option : Word list = #{@conf[:words]}")
-      #diaglogger_info("   Option : Hash Seed = #{@conf[:seed]}")
+      @cmd_list = [
+      	"ps -eo pid,ppid,stime,time,%mem,%cpu,cmd",
+	"cat /proc/meminfo",
+	"netstat -plan",
+	"netstat -s",
+      ]
     end
     
     def run_precheck()
@@ -130,39 +116,35 @@ module Diagtool
       end
       diaglogger_info("[Collect] config file is stored in #{oslog}")
 
-      diaglogger_info("[Collect] Collecting process information...")
-      meminfo = c.collect_ps_eo()
-      diaglogger_info("[Collect] process informationis stored in #{meminfo}")
-
-      diaglogger_info("[Collect] Collecting OS memory information...")
-      meminfo = c.collect_meminfo()
-      diaglogger_info("[Collect] OS memory information is stored in #{meminfo}")
-
       diaglogger_info("[Collect] Collecting date/time information...")
       if system('which chronyc > /dev/null 2>&1')
-        ntp = c.collect_ntp(command="chrony")
+        ntp = c.collect_cmd_output(command="chronyc sources")
+	diaglogger_info("[Collect] date/time information is stored in #{ntp}")
       elsif system('which ntpq > /dev/null 2>&1')
-        ntp = c.collect_ntp(command="ntp")
+        ntp = c.collect_ntp(command="ntpq -p")
+	diaglogger_info("[Collect] date/time information is stored in #{ntp}")
       else
         diaglogger_warn("[Collect] chrony/ntp does not exist. skip collectig date/time information")
       end
-      diaglogger_info("[Collect] date/time information is stored in #{ntp}")
-			
-      diaglogger_info("[Collect] Collecting netstat information...")
-      if system('which netstat > /dev/null 2>&1')
-        netstat_n = c.collect_netstat_plan()
-        netstat_s = c.collect_netstat_s()
-        if @conf[:mask] == 'yes'
-          diaglogger_info("[Mask] Masking netstat file : #{netstat_n}...")
-          netstat_n = m.mask_tdlog(netstat_n, clean = true)
-        end
-        diaglogger_info("[Collect] netstat information is stored in #{netstat_n} and #{netstat_s}")		
-      else
-        diaglogger_warn("[Collect] netstat does not exist. skip collectig netstat")
-      end
       
+      ###
+      #  Correct OS information
+      ###
+      @cmd_list.each { |cmd|
+	diaglogger_info("[Collect] Collecting command output : command = #{cmd}")
+	out = c.collect_cmd_output(cmd)
+	if @conf[:mask] == 'yes'
+          diaglogger_info("[Mask] Masking netstat file : #{out}...")
+          out = m.mask_tdlog(out, clean = true)
+        end
+	diaglogger_info("[Collect] Collecting command output #{cmd.split[0]} stored in #{out}")
+      }
+			
+      ###
+      #  Correct information to be validated
+      ###
       diaglogger_info("[Collect] Collecting systctl information...")
-      sysctl = c.collect_sysctl()
+      sysctl = c.collect_cmd_output("sysctl -a")
       diaglogger_info("[Collect] sysctl information is stored in #{sysctl}")
 			
       diaglogger_info("[Valid] Validating systctl information...")
@@ -176,20 +158,16 @@ module Diagtool
         end
       end
 
-      if system("sh -c 'ulimit -n'")
-      	diaglogger_info("[Collect] Collecting ulimit information...")
-      	ulimit = c.collect_ulimit()
-      	diaglogger_info("[Collect] ulimit information is stored in #{ulimit}")
+      diaglogger_info("[Collect] Collecting ulimit information...")
+      ulimit = c.collect_cmd_output(cmd="sh -c 'ulimit -n'")
+      diaglogger_info("[Collect] ulimit information is stored in #{ulimit}")
 
-      	diaglogger_info("[Valid] Validating ulimit information...")
-      	ret, rec, val = v.valid_ulimit(ulimit)
-      	if ret == true
-          diaglogger_info("[Valid]    ulimit => #{val} is correct (recommendation is >#{rec})")
-        else
-          diaglogger_warn("[Valid]    ulimit => #{val} is incorrect (recommendation is >#{rec})")
-        end
+      diaglogger_info("[Valid] Validating ulimit information...")
+      ret, rec, val = v.valid_ulimit(ulimit)
+      if ret == true
+        diaglogger_info("[Valid]    ulimit => #{val} is correct (recommendation is >#{rec})")
       else
-        diaglogger_warn("[Collect] ulimit command does not exist. skip collectig ulimit")
+        diaglogger_warn("[Valid]    ulimit => #{val} is incorrect (recommendation is >#{rec})")
       end
 
       if @conf[:mask] == 'yes'
@@ -212,7 +190,7 @@ module Diagtool
         diaglogger_info("[Mask] Export mask log file : #{@masklog}")
         m.export_masklog(@masklog)
       end
-      
+
       tar_file = c.compress_output()
       diaglogger_info("[Collect] Generate tar file #{tar_file}")
     end

data/lib/fluent/diagtool/validutils.rb

@@ -33,7 +33,8 @@ module Diagtool
         :net_ipv4_tcp_max_syn_backlog => "8096",
         :net_ipv4_tcp_slow_start_after_idle => "0",
         :net_ipv4_tcp_tw_reuse => "1",
-        :net_ipv4_ip_local_port_range => ["10240", "65535"]}
+        :net_ipv4_ip_local_port_range => ["10240", "65535"]
+      }
       @logger.debug("Initialize Validation Utils:")
       @logger.debug("    Default ulimit: #{@def_ulimit}")
       @logger.debug("    Default sysctl: #{@def_sysctl}")
@@ -57,7 +58,7 @@ module Diagtool
       v = Hash.new { |i,j| i[j] = Hash.new(&h.default_proc) }
       @logger.info("Loading sysctl file: #{sysctl_file}")
       File.readlines(sysctl_file).each{ |line|
-        if line.include?("net")
+        if line.include? "net"
           line_net = line.chomp.gsub(".","_").split("=")
           key = line_net[0].strip.to_sym
           if line_net[1].strip! =~ /\s/
@@ -66,17 +67,19 @@ module Diagtool
             value= line_net[1]
           end
           h[key] = value
-          if @def_sysctl[key] == value
-            @logger.info("#{key} => #{value} is correct")
-            v[key]['value'] = value
-            v[key]['recommend'] = @def_sysctl[key]	
-            v[key]['result'] = "correct"
-          else
-            @logger.warn("#{key} => #{value} is incorrect, should be #{@def_sysctl[key]}")
-            v[key]['value'] = value
-            v[key]['recommend'] = @def_sysctl[key]
-            v[key]['result'] = "incorrect"
-          end
+	  if @def_sysctl.key? key
+            if @def_sysctl[key] == value
+              @logger.info("#{key} => #{value} is correct")
+              v[key]['value'] = value
+              v[key]['recommend'] = @def_sysctl[key]	
+              v[key]['result'] = "correct"
+            else
+              @logger.warn("#{key} => #{value} is incorrect, should be #{@def_sysctl[key]}")
+              v[key]['value'] = value
+              v[key]['recommend'] = @def_sysctl[key]
+              v[key]['result'] = "incorrect"
+            end
+	  end
         end
       }
       if h == @sysctl

data/lib/fluent/diagtool/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Diagtool
-    VERSION = "0.1.7"
+    VERSION = "0.1.8"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-diagtool
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - kubotat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-07 00:00:00.000000000 Z
+date: 2020-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fileutils
@@ -56,8 +56,10 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/diagtool.rb
 - bin/setup
 - bin/word_list_sample
+- exclude_list01
 - exe/diagtool
 - fluent-diagtool.gemspec
 - lib/fluent/diagtool/collectutils.rb