fluent-diagtool 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cec6d985c53b3d147c6d1eb5418ae15983b1bf2ec1bcd15f6c4c035f20a93cf3
-  data.tar.gz: b58b0565d227ccc1f99d0dc3fbb482e03b1cb5faf38c9a5326fe753f0a222efc
+  metadata.gz: 180c7e58ee03d59778640b05042fca84197936b7bdd2b2fff236ef86f0639ef4
+  data.tar.gz: b1ffccfb3a92af08dfa17fe4796e7a229a25d0abe1bf0f1362ac4618bf5c99f0
 SHA512:
-  metadata.gz: c1f5582d4f9835d158f09ee7256771406cf57c8753b6de9170c68d5b4f774ebb7397fce7fe70c8d247eb92bedf9f46a6af6f575b783a4718323b48a2a824b521
-  data.tar.gz: 139b59e1c25cd34e745ba799d5c323c18421d3f5b4b95fbf9c59c63b75291e6663ee9a5cb43cb3cb8ac75ea99a1a495b6c3f9b731960643a93df0fc09e3ddfc1
+  metadata.gz: b482bfc75cda3285976980d2f8a04abfe848dd9bb6ff4992c0760f871f237811fbae361dfb15075482a781ec284974d7471937423e3400f0b576bc3dff03ccf0
+  data.tar.gz: 5810ec1b6f37b5ccf49f53bd8271beb885dde8c7b9e8be33aed38b5d137e8952085aaddbb8b7717e58535bd36decd19e1babde94a2ba78d07ba41d1ffd0cf9c5

data/.gitignore

@@ -0,0 +1,13 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version
+
+test/tmp/

data/AUTHORS

@@ -1 +1 @@
-TOMONORI KUBOTA <tmnr.kubota _at_ gmail.com>
+TOMONORI KUBOTA <tkubota _at_ ctc-america.com>

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    fluent-diagtool (0.1.2)
-      fileutils (~> 1.0.2)
-      json (~> 2.1.0)
+    fluent-diagtool (0.1.4)
+      fileutils (~> 1.0)
+      json (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.3)
-    fileutils (1.0.2)
-    json (2.1.0)
+    fileutils (1.4.1)
+    json (2.3.0)
     rake (12.3.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)

data/README.md

@@ -1,7 +1,7 @@
 # Fluentd Diagnostic Tool
 
-The diagtool enable users to automate the date collection which is required for trouble shooting. The data collected by diagtool include the configuration and log files of the td-agent and diagnostic information of operating system such as network and memory status and stats. In some cases, configuration and log files contains the security sensitive information, such as IP addresses and Hostname. The diagtool also provides the functions to generate mask on IP addresses, Hostname(in FQDN style) and user defined keywords described in the collected data.<br> 
-The scope of data collection:<br>
+The diagtool enable users to automate the date collection which is required for trouble shooting. The data collected by diagtool include the configuration and log files of the td-agent and diagnostic information of operating system such as network and memory status and stats. In some cases, configuration and log files contains the security sensitive information, such as IP addresses and Hostname. The diagtool also provides the functions to generate mask on IP addresses, Hostname(in FQDN style) and user defined keywords described in the collected data.   
+The scope of data collection:  
 - TD Agent information
   - configuration files (*)
   - log files (*)
@@ -11,18 +11,21 @@ The scope of data collection:
   - OS log file
   - OS parameters
     - OS and kernel version
-    - time/date information
-    - maximum number of file descriptor(ulimit)
+    - time/date information(ntp -q/chronyc sources)
+    - maximum number of file descriptor(ulimit -n)
     - kernel network parameters(sysctl)
-  - network conectivity status/stats
-  - memory information
+  - snapshot of current process(ps)
+  - network conectivity status/stats(netstat -plan/netstat -s)
+  - memory information(/proc/meminfo)
 <br>  
-(*) The diagtool automatically gather the path of td-agent configuration files and log files and use them during data collection. 
+(*) The diagtool automatically gather the path of td-agent configuration files and log files from td-agent daemon and use them during data collection. 
 
 ## Prerequisite
+The diagtool provides support for td-agent based installation running on Linux OS. The td-agent is a stable distribution package of Fluentd.  
+The differences between Fluentd and td-agent are described in followed url:  
+https://www.fluentd.org/faqs
 
-
-## Installation
+## Diagtool Installation
 
 ```
 # gem install fluent-diagtool
@@ -114,8 +117,8 @@ NOTE: When user specified the keywork, only the exact match words will be masked
 2020-05-12 18:21:22 -0400: [Diagtool] [INFO] [Collect] Generate tar file /tmp/work1/diagout-20200512182119.tar.gz
 ```
 ## Mask Function
-When run diagtool with mask option, the log of mask is also created in 'mask_{timestamp}.json' file. Users are able to confirm how the mask was generated on each files. 
-<br>
+When run diagtool with mask option, the log of mask is also created in 'mask_{timestamp}.json' file. Users are able to confirm how the mask was generated on each files.  
+The diagtool provides hash-seed option with '-s'. When hash-seed is specified, the mask will be generated with original word and hash-seed so that users could use unique mask value.
 #### Mask sample - IP address: IPv4_{md5hash}
 ```
     "Line112-8": {

data/exe/diagtool

@@ -26,13 +26,21 @@ include Diagtool
 params = {}
 OptionParser.new do |opt|
   opt.banner = "Usage: #{$0} -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {listfile} -s {hash seed}"
+  opt.on('--precheck', 'Run Precheck (Optional)')
   opt.on('-o','--output DIR', String, 'Output directory (Mandatory)')
   opt.on('-m','--mask yes|no', String, 'Enable mask function (Optional : Default=no)')
   opt.on('-w','--word-list word1,word2', Array, 'Provide a list of user-defined words which will to be masked (Optional : Default=None)')
-  opt.on('-f','--word-file listfile', String, 'provide a file which describes a List of user-defined words (Optional : Default=None)')
+  opt.on('-f','--word-file list_file', String, 'provide a file which describes a List of user-defined words (Optional : Default=None)')
   opt.on('-s','--hash-seed seed', String, 'provide a word which will be used when generate the mask (Optional : Default=None)')
+  opt.on('-c','--conf config_file', String, 'provide a full path of td-agent configuration file (Optional : Default=None)')
+  opt.on('-l','--log log_file', String, 'provide a full path of td-agent log file (Optional : Default=None)')
 end.parse!(into: params)
+
 diag = DiagUtils.new(params)
-diag.diagtool()
+if params[:precheck]
+  diag.run_precheck()
+else
+  diag.run_diagtool()
+end
 
 

data/lib/fluent/diagtool/collectutils.rb

@@ -24,17 +24,35 @@ module Diagtool
       @logger = Logger.new(STDOUT, level: log_level, formatter: proc {|severity, datetime, progname, msg|
         "#{datetime}: [Diagutils] [#{severity}] #{msg}\n"
       })
+      @precheck = conf[:precheck]
       @time_format = conf[:time]
       @basedir = conf[:basedir]
       @workdir = conf[:workdir]
       @outdir = conf[:outdir]			
-
+       
       @tdenv = get_tdenv()
-      @tdconf = @tdenv['FLUENT_CONF'].split('/')[-1]
-      @tdconf_path = @tdenv['FLUENT_CONF'].gsub(@tdconf,'')
-      @tdlog =  @tdenv['TD_AGENT_LOG_FILE'].split('/')[-1]
-      @tdlog_path = @tdenv['TD_AGENT_LOG_FILE'].gsub(@tdlog,'')
-
+      if not conf[:tdconf].empty?
+	@tdconf = conf[:tdconf].split('/')[-1]
+        @tdconf_path = conf[:tdconf].gsub(@tdconf,'')
+      elsif
+	if not @tdenv['FLUENT_CONF'].empty?
+      	  @tdconf = @tdenv['FLUENT_CONF'].split('/')[-1]
+      	  @tdconf_path = @tdenv['FLUENT_CONF'].gsub(@tdconf,'')
+	else
+	  raise "The path of td-agent configuration file need to be specified."  if conf[:precheck] == false
+	end
+      end
+      if not conf[:tdlog].empty?
+        @tdlog = conf[:tdlog].split('/')[-1]
+        @tdlog_path = conf[:tdlog].gsub(@tdlog,'')
+      elsif
+        if not @tdenv['TD_AGENT_LOG_FILE'].empty?
+          @tdlog =  @tdenv['TD_AGENT_LOG_FILE'].split('/')[-1]
+          @tdlog_path = @tdenv['TD_AGENT_LOG_FILE'].gsub(@tdlog,'')
+        else
+          raise "The path of td-agent log file need to be specified." if conf[:precheck] == false
+	end
+      end 
       @osenv = get_osenv()
       @oslog_path = '/var/log/'
       @oslog = 'messages'
@@ -57,8 +75,10 @@ module Diagtool
         s = l.split(":")
         os_dict[s[0].chomp.strip] = s[1].chomp.strip
       }
-      File.open(@workdir+'/os_env.output', 'w') do |f|
-        f.puts(stdout)
+      if @precheck == false  # SKip if precheck is true
+        File.open(@outdir+'/os_env.output', 'w') do |f|
+          f.puts(stdout)
+        end
       end
       return os_dict
     end
@@ -66,13 +86,43 @@ module Diagtool
     def get_tdenv()
       stdout, stderr, status = Open3.capture3('systemctl cat td-agent')
       env_dict = {}
-        File.open(@workdir+'/td-agent_env.output', 'w') do |f|
-          f.puts(stdout)
-        end
+      if status.success?
+	if @precheck == false  # SKip if precheck is true
+          File.open(@outdir+'/td-agent_env.output', 'w') do |f|
+            f.puts(stdout)
+          end
+	end  
         stdout.split().each do | l |
-        if l.include?('Environment')
-          env_dict[l.split('=')[1]] = l.split('=')[2]
-        end
+          if l.include?('Environment')
+            env_dict[l.split('=')[1]] = l.split('=')[2]
+          end
+      	end
+      else
+        exe = 'fluentd'
+        stdout, stderr, status = Open3.capture3("ps aux | grep #{exe} | grep -v grep")
+        line = stdout.split(/\n/)
+	log_path = ''
+        conf_path = ''
+        line.each { |l|
+          cmd = l.split.drop(10)
+          i = 0
+          log_pos = 0
+          conf_pos = 0
+          if cmd[-1] != '--under-supervisor'
+            cmd.each { |c|
+              if c.include?("--log") || c.include?("-l")
+                log_pos = i + 1
+                log_path = cmd[log_pos]
+              elsif c.include?("--conf") || c.include?("-c")
+                conf_pos = i + 1
+                conf_path = cmd[conf_pos]
+              end
+              i+=1
+            }
+	  end
+	}
+        env_dict['FLUENT_CONF'] = conf_path
+        env_dict['TD_AGENT_LOG_FILE'] = log_path
       end
       return env_dict
     end
@@ -90,27 +140,34 @@ module Diagtool
     end
     
     def collect_tdconf()
-      FileUtils.mkdir_p(@workdir+@tdconf_path)
-      FileUtils.cp(@tdconf_path+@tdconf, @workdir+@tdconf_path)
-      return @workdir+@tdconf_path+@tdconf
+      target_dir = @workdir+@tdconf_path
+      FileUtils.mkdir_p(target_dir)
+      FileUtils.cp(@tdconf_path+@tdconf, target_dir)
+      return target_dir+@tdconf
     end
     
     def collect_tdlog()
-      FileUtils.mkdir_p(@workdir+@tdlog_path)
-      FileUtils.cp_r(@tdlog_path, @workdir+@oslog_path)
-      return Dir.glob(@workdir+@tdlog_path+@tdlog+'*')
+      target_dir = @workdir+@tdlog_path
+      p target_dir
+      FileUtils.mkdir_p(target_dir)
+      Dir.glob(@tdlog_path+@tdlog+'*').each{ |f| 
+        FileUtils.cp(f, target_dir)
+      }
+      return Dir.glob(target_dir+@tdlog+'*')
     end
     
     def collect_sysctl()
-      FileUtils.mkdir_p(@workdir+@sysctl_path)
-      FileUtils.cp(@sysctl_path+@sysctl, @workdir+@sysctl_path)
-      return @workdir+@sysctl_path+@sysctl
+      target_dir = @workdir+@sysctl_path
+      FileUtils.mkdir_p(target_dir)
+      FileUtils.cp(@sysctl_path+@sysctl, target_dir)
+      return target_dir+@sysctl
     end
     
     def collect_oslog()
-      FileUtils.mkdir_p(@workdir+@oslog_path)
-      FileUtils.cp(@oslog_path+@oslog, @workdir+@oslog_path)
-      return @workdir+@oslog_path+@oslog
+      target_dir = @workdir+@oslog_path
+      FileUtils.mkdir_p(target_dir)
+      FileUtils.cp(@oslog_path+@oslog, target_dir)
+      return target_dir+@oslog
     end
     
     def collect_ulimit()

data/lib/fluent/diagtool/diagutils.rb

@@ -27,14 +27,62 @@ module Diagtool
       time = Time.new
       @time_format = time.strftime("%Y%m%d%0k%M%0S")
       @conf = parse_diagconf(params)
+      #@conf[:time] = @time_format
+      #@conf[:workdir] = @conf[:basedir] + '/' + @time_format
+      #@conf[:outdir] = @conf[:workdir] + '/output'
+
+      #FileUtils.mkdir_p(@conf[:workdir])
+      #FileUtils.mkdir_p(@conf[:outdir])
+      
+      #diaglog = @conf[:workdir] + '/diagtool.output'
+      #@masklog = './mask_' + @time_format + '.json'
+      #@logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+      #  "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+      #})
+      #@logger_file = Logger.new(diaglog, formatter: proc {|severity, datetime, progname, msg|
+      #  "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+      #})
+      #diaglogger_info("Parsing command options...")
+      #diaglogger_info("   Option : Output directory = #{@conf[:basedir]}")
+      #diaglogger_info("   Option : Mask = #{@conf[:mask]}")
+      #diaglogger_info("   Option : Word list = #{@conf[:words]}")
+      #diaglogger_info("   Option : Hash Seed = #{@conf[:seed]}")
+    end
+    
+    def run_precheck()
+      prechecklog = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+        "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+      })
+      loglevel = 'WARN'
+      c = CollectUtils.new(@conf, loglevel)
+      c_env = c.export_env()
+      prechecklog.info("[Precheck] Check OS parameters...")
+      prechecklog.info("[Precheck]    operating system = #{c_env[:os]}")
+      prechecklog.info("[Precheck]    kernel version = #{c_env[:kernel]}")
+      prechecklog.info("[Precheck] Check td-agent parameters...")
+      prechecklog.info("[Precheck]    td-agent conf path = #{c_env[:tdconf_path]}")
+      prechecklog.info("[Precheck]    td-agent conf file = #{c_env[:tdconf]}")
+      prechecklog.info("[Precheck]    td-agent log path = #{c_env[:tdlog_path]}")
+      prechecklog.info("[Precheck]    td-agent log = #{c_env[:tdlog]}")
+      if c_env[:tdconf_path] == nil || c_env[:tdconf] == nil
+	prechecklog.warn("[Precheck]    can not find td-agent conf path: please run diagtool command with -c /path/to/<td-agent conf file>")
+      end
+      if c_env[:tdlog_path] == nil || c_env[:tdlog] == nil
+        prechecklog.warn("[Precheck]    can not find td-agent log path: please run diagtool command with -l /path/to/<td-agent log file>")
+      end
+      if c_env[:tdconf_path] != nil && c_env[:tdconf] != nil && c_env[:tdlog_path] != nil && c_env[:tdlog] != nil
+	 prechecklog.info("[Precheck] Precheck completed. You can run diagtool command without -c and -l options")
+      end
+    end
+
+    def run_diagtool()
       @conf[:time] = @time_format
       @conf[:workdir] = @conf[:basedir] + '/' + @time_format
       @conf[:outdir] = @conf[:workdir] + '/output'
-      
       FileUtils.mkdir_p(@conf[:workdir])
       FileUtils.mkdir_p(@conf[:outdir])
-      
       diaglog = @conf[:workdir] + '/diagtool.output'
+
       @masklog = './mask_' + @time_format + '.json'
       @logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
         "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
@@ -47,9 +95,7 @@ module Diagtool
       diaglogger_info("   Option : Mask = #{@conf[:mask]}")
       diaglogger_info("   Option : Word list = #{@conf[:words]}")
       diaglogger_info("   Option : Hash Seed = #{@conf[:seed]}")
-    end
-    
-    def diagtool()
+
       loglevel = 'WARN'
       diaglogger_info("Initializing parameters...")
       c = CollectUtils.new(@conf, loglevel)
@@ -167,16 +213,23 @@ module Diagtool
 
     def parse_diagconf(params)
       options = {
-        :basedir => '', :mask => '', :words => [], :wfile => '', :seed => ''
+        :precheck => '', :basedir => '', :mask => '', :words => [], :wfile => '', :seed => '', :tdconf =>'', :tdlog => ''
       }
-      if params[:output] != nil
-        if Dir.exist?(params[:output])
-          options[:basedir] = params[:output]
+      if params[:precheck]
+        options[:precheck] = params[:precheck]
+      else
+        options[:precheck] = false
+      end
+      if options[:precheck] == false
+        if params[:output] != nil
+          if Dir.exist?(params[:output])
+            options[:basedir] = params[:output]
+          else
+            raise "output directory '#{basedir}' does not exist"
+          end
         else
-          raise "output directory '#{basedir}' does not exist"
+          raise "output directory '-o' must be specified"
         end
-      else
-        raise "output directory '-o' must be specified"
       end
       if params[:mask] == nil
         options[:mask] = 'no'
@@ -200,6 +253,25 @@ module Diagtool
       end
       options[:words] = options[:words].uniq 
       options[:seed] = params[:"hash-seed"] if params[:"hash-seed"] != nil
+      
+      if params[:conf] != nil
+        f = params[:conf]
+        if File.exist?(f)
+	  options[:tdconf] = params[:conf]
+        else
+          raise "#{params[:conf]} : No such file or directory"
+        end
+      end
+
+      if params[:log] != nil
+        f = params[:log]
+        if File.exist?(f)
+          options[:tdlog] = params[:log]
+        else
+          raise "#{params[:log]} : No such file or directory"
+        end
+      end
+
       return options	
     end
     

data/lib/fluent/diagtool/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Diagtool
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-diagtool
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - kubotat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2020-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fileutils
@@ -48,6 +48,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - AUTHORS
 - Gemfile
 - Gemfile.lock