fluent-diagtool 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9adde52501cea6ad1da43f3043f88eca6acfc6b2d9a1e50b010a8b56b159d7f8
-  data.tar.gz: f08409b34fb1c6ef681a639151638d6a1c62f39f525bb331f5bc9031bcf12386
+  metadata.gz: cec6d985c53b3d147c6d1eb5418ae15983b1bf2ec1bcd15f6c4c035f20a93cf3
+  data.tar.gz: b58b0565d227ccc1f99d0dc3fbb482e03b1cb5faf38c9a5326fe753f0a222efc
 SHA512:
-  metadata.gz: 0e6a14f2b174d5910904bcbfcf494d61e5b432aac9b24a32791735c21d643cf07da3a15f0438f50f3ae1aea1d279173316204ea5410a8a6389c15b5989995d65
-  data.tar.gz: 9d96a8edd787fd4a3fb166d12120dea1f6d137577479fe70f3a355c55890508acaa8045d07bcea76ff06c67e6dc2bd8a885ecf77bf377f19dbc45be233f5a756
+  metadata.gz: c1f5582d4f9835d158f09ee7256771406cf57c8753b6de9170c68d5b4f774ebb7397fce7fe70c8d247eb92bedf9f46a6af6f575b783a4718323b48a2a824b521
+  data.tar.gz: 139b59e1c25cd34e745ba799d5c323c18421d3f5b4b95fbf9c59c63b75291e6663ee9a5cb43cb3cb8ac75ea99a1a495b6c3f9b731960643a93df0fc09e3ddfc1

data/Gemfile.lock

@@ -1,12 +1,16 @@
 PATH
   remote: .
   specs:
-    fluent-diagtool (0.1.0)
+    fluent-diagtool (0.1.2)
+      fileutils (~> 1.0.2)
+      json (~> 2.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.3)
+    fileutils (1.0.2)
+    json (2.1.0)
     rake (12.3.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)

data/README.md

@@ -22,10 +22,24 @@ The scope of data collection:<br>
 ## Prerequisite
 
 
+## Installation
+
+```
+# gem install fluent-diagtool
+Fetching: fileutils-1.0.2.gem (100%)
+Successfully installed fileutils-1.0.2
+Fetching: json-2.1.0.gem (100%)
+Building native extensions. This could take a while...
+Successfully installed json-2.1.0
+Fetching: fluent-diagtool-0.1.2.gem (100%)
+Successfully installed fluent-diagtool-0.1.2
+3 gems installed
+```
+
 ## Usage
 ```
-# ruby diagtool.rb --help
-Usage: diagtool.rb -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {listfile} -s {hash seed}
+# diagtool --help
+Usage: /usr/local/bin/diagtool -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {listfile} -s {hash seed}
     -o, --output DIR                 Output directory (Mandatory)
     -m, --mask yes|no                Enable mask function (Optional : Default=no)
     -w, --word-list word1,word2      Provide a list of user-defined words which will to be masked (Optional : Default=None)
@@ -43,7 +57,7 @@ NOTE: When user specified the keywork, only the exact match words will be masked
 
 #### Command sample:
 ```
-# ruby diagtool.rb -o /tmp/work1 -w passwd1,passwd2 -f word_list_sample -m yes
+# diagtool -o /tmp/work1 -w passwd1,passwd2 -f word_list_sample -m yes
 2020-05-12 18:21:19 -0400: [Diagtool] [INFO] Parsing command options...
 2020-05-12 18:21:19 -0400: [Diagtool] [INFO]    Option : Output directory = /tmp/work1
 2020-05-12 18:21:19 -0400: [Diagtool] [INFO]    Option : Mask = yes

data/fluent-diagtool.gemspec

@@ -19,6 +19,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency("fileutils", ["~> 1.0.2"])
-  spec.add_runtime_dependency("json", ["~> 2.1.0"])
+  spec.add_runtime_dependency("fileutils", ["~> 1.0"])
+  spec.add_runtime_dependency("json", ["~> 2.1"])
 end

data/lib/fluent/diagtool/collectutils.rb

@@ -25,9 +25,10 @@ module Diagtool
         "#{datetime}: [Diagutils] [#{severity}] #{msg}\n"
       })
       @time_format = conf[:time]
-      @output_dir = conf[:output_dir]
+      @basedir = conf[:basedir]
       @workdir = conf[:workdir]
-			
+      @outdir = conf[:outdir]			
+
       @tdenv = get_tdenv()
       @tdconf = @tdenv['FLUENT_CONF'].split('/')[-1]
       @tdconf_path = @tdenv['FLUENT_CONF'].gsub(@tdconf,'')
@@ -48,6 +49,7 @@ module Diagtool
       @logger.info("    td-agent log path = #{@tdlog_path}")
       @logger.info("    td-agent log = #{@tdlog}")
     end
+    
     def get_osenv()
       stdout, stderr, status = Open3.capture3('hostnamectl')
       os_dict = {}
@@ -60,6 +62,7 @@ module Diagtool
       end
       return os_dict
     end
+    
     def get_tdenv()
       stdout, stderr, status = Open3.capture3('systemctl cat td-agent')
       env_dict = {}
@@ -73,6 +76,7 @@ module Diagtool
       end
       return env_dict
     end
+    
     def export_env()
       env = {
         :os => @osenv['Operating System'],
@@ -84,60 +88,78 @@ module Diagtool
       }
       return env
     end
+    
     def collect_tdconf()
       FileUtils.mkdir_p(@workdir+@tdconf_path)
       FileUtils.cp(@tdconf_path+@tdconf, @workdir+@tdconf_path)
       return @workdir+@tdconf_path+@tdconf
     end
+    
     def collect_tdlog()
       FileUtils.mkdir_p(@workdir+@tdlog_path)
       FileUtils.cp_r(@tdlog_path, @workdir+@oslog_path)
       return Dir.glob(@workdir+@tdlog_path+@tdlog+'*')
     end
+    
     def collect_sysctl()
       FileUtils.mkdir_p(@workdir+@sysctl_path)
       FileUtils.cp(@sysctl_path+@sysctl, @workdir+@sysctl_path)
       return @workdir+@sysctl_path+@sysctl
     end
+    
     def collect_oslog()
       FileUtils.mkdir_p(@workdir+@oslog_path)
       FileUtils.cp(@oslog_path+@oslog, @workdir+@oslog_path)
       return @workdir+@oslog_path+@oslog
     end
+    
     def collect_ulimit()
-      output = @workdir+'/ulimit_n.output'
+      output = @outdir+'/ulimit_n.output'
       stdout, stderr, status = Open3.capture3("ulimit -n")
       File.open(output, 'w') do |f|
         f.puts(stdout)
       end
       return output
     end
+   
+    def collect_ps_eo()
+      output = @outdir+'/ps_eo.output'
+      stdout, stderr, status = Open3.capture3("ps -eo pid,ppid,stime,time,%mem,%cpu,cmd")
+      File.open(output, 'w') do |f|
+        f.puts(stdout)
+      end
+      return output
+    end
+
     def collect_meminfo()
-      output = @workdir+'/meminfo.output'
+      output = @outdir+'/meminfo.output'
       stdout, stderr, status = Open3.capture3("cat /proc/meminfo")
       File.open(output, 'w') do |f|
         f.puts(stdout)
       end
       return output
     end
-    def collect_netstat_n()
-      output = @workdir+'/netstat_n.output'
-      stdout, stderr, status = Open3.capture3("netstat -n")
+    
+    def collect_netstat_plan()
+      output = @outdir+'/netstat_plan.output'
+      stdout, stderr, status = Open3.capture3("netstat -plan")
       File.open(output, 'w') do |f|
         f.puts(stdout)
       end
       return output
     end
+    
     def collect_netstat_s()
-      output = @workdir+'/netstat_s.output'
+      output = @outdir+'/netstat_s.output'
       stdout, stderr, status = Open3.capture3("netstat -s")
       File.open(output, 'w') do |f|
         f.puts(stdout)
       end
       return output
     end
+    
     def collect_ntp(command)
-      output = @workdir+'/ntp_info.output'
+      output = @outdir+'/ntp_info.output'
       stdout_date, stderr_date, status_date = Open3.capture3("date")
       stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("chronyc sources") if command == "chrony"
       stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("ntpq -p") if command == "ntp"
@@ -147,19 +169,21 @@ module Diagtool
       end
       return output
     end
+    
     def collect_tdgems()
-      output = @workdir+'/tdgem_list.output'
+      output = @outdir+'/tdgem_list.output'
       stdout, stderr, status = Open3.capture3("td-agent-gem list | grep fluent")
       File.open(output, 'w') do |f|
         f.puts(stdout)
       end
       return output
     end
+    
     def compress_output()
-      Dir.chdir(@output_dir)
+      Dir.chdir(@basedir)
       tar_file = 'diagout-'+@time_format+'.tar.gz'
       stdout, stderr, status = Open3.capture3("tar cvfz #{tar_file} #{@time_format}")
-      return @output_dir + '/' + tar_file
+      return @basedir + '/' + tar_file
     end
   end
 end

data/lib/fluent/diagtool/diagutils.rb

@@ -28,8 +28,12 @@ module Diagtool
       @time_format = time.strftime("%Y%m%d%0k%M%0S")
       @conf = parse_diagconf(params)
       @conf[:time] = @time_format
-      @conf[:workdir] = @conf[:output_dir] + '/' + @time_format
+      @conf[:workdir] = @conf[:basedir] + '/' + @time_format
+      @conf[:outdir] = @conf[:workdir] + '/output'
+      
       FileUtils.mkdir_p(@conf[:workdir])
+      FileUtils.mkdir_p(@conf[:outdir])
+      
       diaglog = @conf[:workdir] + '/diagtool.output'
       @masklog = './mask_' + @time_format + '.json'
       @logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
@@ -39,11 +43,12 @@ module Diagtool
         "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
       })
       diaglogger_info("Parsing command options...")
-      diaglogger_info("   Option : Output directory = #{@conf[:output_dir]}")
+      diaglogger_info("   Option : Output directory = #{@conf[:basedir]}")
       diaglogger_info("   Option : Mask = #{@conf[:mask]}")
       diaglogger_info("   Option : Word list = #{@conf[:words]}")
       diaglogger_info("   Option : Hash Seed = #{@conf[:seed]}")
     end
+    
     def diagtool()
       loglevel = 'WARN'
       diaglogger_info("Initializing parameters...")
@@ -78,10 +83,14 @@ module Diagtool
         oslog = m.mask_tdlog(oslog, clean = true)
       end
       diaglogger_info("[Collect] config file is stored in #{oslog}")
-			
+
+      diaglogger_info("[Collect] Collecting process information...")
+      meminfo = c.collect_ps_eo()
+      diaglogger_info("[Collect] process informationis stored in #{meminfo}")
+
       diaglogger_info("[Collect] Collecting OS memory information...")
       meminfo = c.collect_meminfo()
-      diaglogger_info("[Collect] config file is stored in #{meminfo}")
+      diaglogger_info("[Collect] OS memory information is stored in #{meminfo}")
 
       diaglogger_info("[Collect] Collecting date/time information...")
       if system('which chronyc > /dev/null 2>&1')
@@ -95,7 +104,7 @@ module Diagtool
 			
       diaglogger_info("[Collect] Collecting netstat information...")
       if system('which netstat > /dev/null 2>&1')
-        netstat_n = c.collect_netstat_n()
+        netstat_n = c.collect_netstat_plan()
         netstat_s = c.collect_netstat_s()
         if @conf[:mask] == 'yes'
           diaglogger_info("[Mask] Masking netstat file : #{netstat_n}...")
@@ -158,17 +167,13 @@ module Diagtool
 
     def parse_diagconf(params)
       options = {
-        :output_dir => '',
-        :mask => 'no',
-        :words => [],
-        :wfile => '',
-        :seed => ''
+        :basedir => '', :mask => '', :words => [], :wfile => '', :seed => ''
       }
       if params[:output] != nil
         if Dir.exist?(params[:output])
-          options[:output_dir] = params[:output]
+          options[:basedir] = params[:output]
         else
-          raise "output directory '#{output_dir}' does not exist"
+          raise "output directory '#{basedir}' does not exist"
         end
       else
         raise "output directory '-o' must be specified"
@@ -197,18 +202,22 @@ module Diagtool
       options[:seed] = params[:"hash-seed"] if params[:"hash-seed"] != nil
       return options	
     end
+    
     def diaglogger_debug(str)
       @logger.debug(str)
       @logger_file.debug(str)
     end
+    
     def diaglogger_info(str)
       @logger.info(str)
       @logger_file.info(str)
     end
+    
     def diaglogger_warn(str)
       @logger.warn(str)
       @logger_file.warn(str)
     end
+    
     def diaglogger_error(str)
       @logger.error(str)
       @logger_file.error(str)

data/lib/fluent/diagtool/maskutils.rb

@@ -29,13 +29,10 @@ module Diagtool
       })
       @logger.debug("Initialize Maskutils: sanitized word = #{conf[:words]}")
       @hash_seed = conf[:seed]
-      @id = {
-        :fid =>'',
-        :lid =>'',
-        :cid =>''
-      }
+      @id = {}
       @masklog = Hash.new { |h,k| h[k] = Hash.new(&h.default_proc) }
     end
+    
     def mask_tdlog(input_file, clean)
       line_id = 0
       f = File.open(input_file+'.mask', 'w')
@@ -51,6 +48,7 @@ module Diagtool
       FileUtils.rm(input_file) if clean == true
       return input_file+'.mask'
     end
+    
     def mask_tdlog_gz(input_file, clean)
       line_id = 0
       f = File.open(input_file+'.mask', 'w')
@@ -68,6 +66,7 @@ module Diagtool
       FileUtils.rm(input_file) if clean == true
       return input_file+'.mask'
     end
+    
     def mask_tdlog_inspector(line)
       i = 0
       contents=[]
@@ -157,6 +156,7 @@ module Diagtool
       @logger.debug("Masked Line: #{line_masked}")
       return line_masked
     end
+    
     def mask_direct_pattern(str)
       is_mask = false
       if str.include?(">")
@@ -173,6 +173,7 @@ module Diagtool
       end
       return is_mask, str_mask
     end	
+    
     def mask_url_pattern(str)
       is_mask = false
       url = str.split('://')
@@ -215,6 +216,7 @@ module Diagtool
       str_mask << ":" if str.end_with?(':')
       return is_mask, str_mask
     end
+    
     def mask_equal_pattern(str)
       is_mask = false
       l = str.split('=') ## Mask host=<address:ip/hostname> or bind=<address: ip/hostname>
@@ -228,6 +230,7 @@ module Diagtool
       str_mask = l.join('=')
       return is_mask, str_mask
     end
+    
     def mask_colon_pattern(str)
       is_mask = false
       l = str.split(':')
@@ -242,6 +245,7 @@ module Diagtool
       str_mask << ":" if str.end_with?(':')
       return is_mask, str_mask
     end
+    
     def mask_slash_pattern(str)
       is_mask = false
       l = str.split('/')
@@ -256,14 +260,17 @@ module Diagtool
       str_mask << ":" if str.end_with?(':')
       return is_mask, str_mask
     end
+    
     def is_ipv4?(str)
       !!(str =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)
     end
+    
     def is_fqdn?(str)
       #!!(str =~ /^\b((?=[a-z0-9-]{1,63}\.)[a-z0-9]+(-[a-z0-9]+)*\.)+([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/)
       !!(str =~ /^\b(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.){2,}([A-Za-z]|[A-Za-z][A-Za-z\-]*[A-Za-z]){2,}$/)
       #!!(str =~ /^\b(?=^.{1,254}$)(^(?:(?!\d+\.)[a-zA-Z0-9_\-]{1,63}\.?)+(?:[a-zA-Z]{2,})$)/)
     end
+    
     def is_words?(str)
       value = false
       @words.each do | l |
@@ -274,6 +281,7 @@ module Diagtool
       end
       return value
     end
+    
     def mask_ipv4_fqdn_words(str)
       str = str.to_s
       mtype = ''
@@ -299,11 +307,13 @@ module Diagtool
       end
       return is_mask, str, str_mask
     end
+    
     def put_masklog(str, str_mask)
       uid = "Line#{@id[:lid]}-#{@id[:cid]}"
       @masklog[@id[:fid]][uid]['original'] = str
       @masklog[@id[:fid]][uid]['mask'] = str_mask
     end
+    
     def export_masklog(output_file)
       masklog_json = JSON.pretty_generate(@masklog)
       File.open(output_file, 'w') do |f|

data/lib/fluent/diagtool/validutils.rb

@@ -38,6 +38,7 @@ module Diagtool
       @logger.debug("    Default ulimit: #{@def_ulimit}")
       @logger.debug("    Default sysctl: #{@def_sysctl}")
     end
+    
     def valid_ulimit(ulimit_file)
       @logger.info("Loading ulimit file: #{ulimit_file}")
       File.readlines(ulimit_file).each { |line|
@@ -50,6 +51,7 @@ module Diagtool
         end 
       }
     end
+    
     def valid_sysctl(sysctl_file)
       h = Hash.new()
       v = Hash.new { |i,j| i[j] = Hash.new(&h.default_proc) }

data/lib/fluent/diagtool/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Diagtool
-    VERSION = "0.1.2"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-diagtool
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - kubotat
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '2.1'
 description: Bringing productivity of trouble shooting to the next level  by automating
   collection of Fluentd configurations, settings and OS parameters as well as masking
   sensitive information in logs and configurations.
@@ -55,7 +55,6 @@ files:
 - README.md
 - Rakefile
 - bin/console
-- bin/diagtool.rb
 - bin/setup
 - bin/word_list_sample
 - exe/diagtool

data/bin/diagtool.rb

@@ -1,37 +0,0 @@
-#
-# Fluentd
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-#!/usr/bin/ruby
-require 'optparse'
-require '../lib/collectutils'
-require '../lib/maskutils'
-require '../lib/validutils'
-require '../lib/diagutils'
-include Diagtool
-
-params = {}
-OptionParser.new do |opt|
-  opt.banner = "Usage: #{$0} -o OUTPUT_DIR -m {yes | no} -w {word1,[word2...]} -f {listfile} -s {hash seed}"
-  opt.on('-o','--output DIR', String, 'Output directory (Mandatory)')
-  opt.on('-m','--mask yes|no', String, 'Enable mask function (Optional : Default=no)')
-  opt.on('-w','--word-list word1,word2', Array, 'Provide a list of user-defined words which will to be masked (Optional : Default=None)')
-  opt.on('-f','--word-file listfile', String, 'provide a file which describes a List of user-defined words (Optional : Default=None)')
-  opt.on('-s','--hash-seed seed', String, 'provide a word which will be used when generate the mask (Optional : Default=None)')
-end.parse!(into: params)
-diag = DiagUtils.new(params)
-diag.diagtool()
-
-