fluent-diagtool 0.1.1

data/lib/fluent/diagtool/collectutils.rb

@@ -0,0 +1,165 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fileutils'
+require 'open3'
+require 'logger'
+
+module Diagtool
+  class CollectUtils
+    def initialize(conf, log_level)
+      @logger = Logger.new(STDOUT, level: log_level, formatter: proc {|severity, datetime, progname, msg|
+        "#{datetime}: [Diagutils] [#{severity}] #{msg}\n"
+      })
+      @time_format = conf[:time]
+      @output_dir = conf[:output_dir]
+      @workdir = conf[:workdir]
+			
+      @tdenv = get_tdenv()
+      @tdconf = @tdenv['FLUENT_CONF'].split('/')[-1]
+      @tdconf_path = @tdenv['FLUENT_CONF'].gsub(@tdconf,'')
+      @tdlog =  @tdenv['TD_AGENT_LOG_FILE'].split('/')[-1]
+      @tdlog_path = @tdenv['TD_AGENT_LOG_FILE'].gsub(@tdlog,'')
+
+      @osenv = get_osenv()
+      @oslog_path = '/var/log/'
+      @oslog = 'messages'
+      @sysctl_path = '/etc/'
+      @sysctl = 'sysctl.conf'        
+
+      @logger.info("Loading the environment parameters...")
+      @logger.info("    operating system = #{@osenv['Operating System']}")
+      @logger.info("    kernel version = #{@osenv['Kernel']}")
+      @logger.info("    td-agent conf path = #{@tdconf_path}")
+      @logger.info("    td-agent conf file = #{@tdconf}")
+      @logger.info("    td-agent log path = #{@tdlog_path}")
+      @logger.info("    td-agent log = #{@tdlog}")
+    end
+    def get_osenv()
+      stdout, stderr, status = Open3.capture3('hostnamectl')
+      os_dict = {}
+      stdout.each_line { |l|
+        s = l.split(":")
+        os_dict[s[0].chomp.strip] = s[1].chomp.strip
+      }
+      File.open(@workdir+'/os_env.output', 'w') do |f|
+        f.puts(stdout)
+      end
+      return os_dict
+    end
+    def get_tdenv()
+      stdout, stderr, status = Open3.capture3('systemctl cat td-agent')
+      env_dict = {}
+        File.open(@workdir+'/td-agent_env.output', 'w') do |f|
+          f.puts(stdout)
+        end
+        stdout.split().each do | l |
+        if l.include?('Environment')
+          env_dict[l.split('=')[1]] = l.split('=')[2]
+        end
+      end
+      return env_dict
+    end
+    def export_env()
+      env = {
+        :os => @osenv['Operating System'],
+        :kernel => @osenv['Kernel'],
+        :tdconf => @tdconf,
+        :tdconf_path => @tdconf_path,
+        :tdlog => @tdlog,
+        :tdlog_path => @tdlog_path
+      }
+      return env
+    end
+    def collect_tdconf()
+      FileUtils.mkdir_p(@workdir+@tdconf_path)
+      FileUtils.cp(@tdconf_path+@tdconf, @workdir+@tdconf_path)
+      return @workdir+@tdconf_path+@tdconf
+    end
+    def collect_tdlog()
+      FileUtils.mkdir_p(@workdir+@tdlog_path)
+      FileUtils.cp_r(@tdlog_path, @workdir+@oslog_path)
+      return Dir.glob(@workdir+@tdlog_path+@tdlog+'*')
+    end
+    def collect_sysctl()
+      FileUtils.mkdir_p(@workdir+@sysctl_path)
+      FileUtils.cp(@sysctl_path+@sysctl, @workdir+@sysctl_path)
+      return @workdir+@sysctl_path+@sysctl
+    end
+    def collect_oslog()
+      FileUtils.mkdir_p(@workdir+@oslog_path)
+      FileUtils.cp(@oslog_path+@oslog, @workdir+@oslog_path)
+      return @workdir+@oslog_path+@oslog
+    end
+    def collect_ulimit()
+      output = @workdir+'/ulimit_n.output'
+      stdout, stderr, status = Open3.capture3("ulimit -n")
+      File.open(output, 'w') do |f|
+        f.puts(stdout)
+      end
+      return output
+    end
+    def collect_meminfo()
+      output = @workdir+'/meminfo.output'
+      stdout, stderr, status = Open3.capture3("cat /proc/meminfo")
+      File.open(output, 'w') do |f|
+        f.puts(stdout)
+      end
+      return output
+    end
+    def collect_netstat_n()
+      output = @workdir+'/netstat_n.output'
+      stdout, stderr, status = Open3.capture3("netstat -n")
+      File.open(output, 'w') do |f|
+        f.puts(stdout)
+      end
+      return output
+    end
+    def collect_netstat_s()
+      output = @workdir+'/netstat_s.output'
+      stdout, stderr, status = Open3.capture3("netstat -s")
+      File.open(output, 'w') do |f|
+        f.puts(stdout)
+      end
+      return output
+    end
+    def collect_ntp(command)
+      output = @workdir+'/ntp_info.output'
+      stdout_date, stderr_date, status_date = Open3.capture3("date")
+      stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("chronyc sources") if command == "chrony"
+      stdout_ntp, stderr_ntp, status_ntp = Open3.capture3("ntpq -p") if command == "ntp"
+      File.open(output, 'w') do |f|
+        f.puts(stdout_date)
+        f.puts(stdout_ntp)
+      end
+      return output
+    end
+    def collect_tdgems()
+      output = @workdir+'/tdgem_list.output'
+      stdout, stderr, status = Open3.capture3("td-agent-gem list | grep fluent")
+      File.open(output, 'w') do |f|
+        f.puts(stdout)
+      end
+      return output
+    end
+    def compress_output()
+      Dir.chdir(@output_dir)
+      tar_file = 'diagout-'+@time_format+'.tar.gz'
+      stdout, stderr, status = Open3.capture3("tar cvfz #{tar_file} #{@time_format}")
+      return @output_dir + '/' + tar_file
+    end
+  end
+end

data/lib/fluent/diagtool/diagutils.rb

@@ -0,0 +1,217 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'logger'
+require 'fileutils'
+require 'fluent/diagtool/collectutils'
+require 'fluent/diagtool/maskutils'
+require 'fluent/diagtool/validutils'
+include Diagtool
+
+module Diagtool
+  class DiagUtils
+    def initialize(params)
+      time = Time.new
+      @time_format = time.strftime("%Y%m%d%0k%M%0S")
+      @conf = parse_diagconf(params)
+      @conf[:time] = @time_format
+      @conf[:workdir] = @conf[:output_dir] + '/' + @time_format
+      FileUtils.mkdir_p(@conf[:workdir])
+      diaglog = @conf[:workdir] + '/diagtool.output'
+      @masklog = './mask_' + @time_format + '.json'
+      @logger = Logger.new(STDOUT, formatter: proc {|severity, datetime, progname, msg|
+        "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+      })
+      @logger_file = Logger.new(diaglog, formatter: proc {|severity, datetime, progname, msg|
+        "#{datetime}: [Diagtool] [#{severity}] #{msg}\n"
+      })
+      diaglogger_info("Parsing command options...")
+      diaglogger_info("   Option : Output directory = #{@conf[:output_dir]}")
+      diaglogger_info("   Option : Mask = #{@conf[:mask]}")
+      diaglogger_info("   Option : Word list = #{@conf[:words]}")
+      diaglogger_info("   Option : Hash Seed = #{@conf[:seed]}")
+    end
+    def diagtool()
+      loglevel = 'WARN'
+      diaglogger_info("Initializing parameters...")
+      c = CollectUtils.new(@conf, loglevel)
+      c_env = c.export_env()
+      diaglogger_info("[Collect] Loading the environment parameters...")
+      diaglogger_info("[Collect]    operating system = #{c_env[:os]}")
+      diaglogger_info("[Collect]    kernel version = #{c_env[:kernel]}")
+      diaglogger_info("[Collect]    td-agent conf path = #{c_env[:tdconf_path]}")
+      diaglogger_info("[Collect]    td-agent conf file = #{c_env[:tdconf]}")
+      diaglogger_info("[Collect]    td-agent log path = #{c_env[:tdlog_path]}")
+      diaglogger_info("[Collect]    td-agent log = #{c_env[:tdlog]}")
+      m = MaskUtils.new(@conf, loglevel)
+      v = ValidUtils.new(loglevel)
+							
+      diaglogger_info("[Collect] Collecting log files of td-agent...")
+      tdlog = c.collect_tdlog()
+      diaglogger_info("[Collect] log files of td-agent are stored in #{tdlog}")
+
+      diaglogger_info("[Collect] Collecting config file of td-agent...")
+      tdconf = c.collect_tdconf()
+      diaglogger_info("[Collect] config file is stored in #{tdconf}")
+
+      diaglogger_info("[Collect] Collecting td-agent gem information...")
+      tdgem = c.collect_tdgems()
+      diaglogger_info("[Collect] td-agent gem information is stored in #{tdgem}")
+
+      diaglogger_info("[Collect] Collecting config file of OS log...")
+      oslog = c.collect_oslog()
+      if @conf[:mask] == 'yes'
+        diaglogger_info("[Mask] Masking OS log file : #{oslog}...")
+        oslog = m.mask_tdlog(oslog, clean = true)
+      end
+      diaglogger_info("[Collect] config file is stored in #{oslog}")
+			
+      diaglogger_info("[Collect] Collecting OS memory information...")
+      meminfo = c.collect_meminfo()
+      diaglogger_info("[Collect] config file is stored in #{meminfo}")
+
+      diaglogger_info("[Collect] Collecting date/time information...")
+      if system('which chronyc > /dev/null 2>&1')
+        ntp = c.collect_ntp(command="chrony")
+      elsif system('which ntpq > /dev/null 2>&1')
+        ntp = c.collect_ntp(command="ntp")
+      else
+        diaglogger_warn("[Collect] chrony/ntp does not exist. skip collectig date/time information")
+      end
+      diaglogger_info("[Collect] date/time information is stored in #{ntp}")
+			
+      diaglogger_info("[Collect] Collecting netstat information...")
+      if system('which netstat > /dev/null 2>&1')
+        netstat_n = c.collect_netstat_n()
+        netstat_s = c.collect_netstat_s()
+        if @conf[:mask] == 'yes'
+          diaglogger_info("[Mask] Masking netstat file : #{netstat_n}...")
+          netstat_n = m.mask_tdlog(netstat_n, clean = true)
+        end
+        diaglogger_info("[Collect] netstat information is stored in #{netstat_n} and #{netstat_s}")		
+      else
+        diaglogger_warn("[Collect] netstat does not exist. skip collectig netstat")
+      end
+      
+      diaglogger_info("[Collect] Collecting systctl information...")
+      sysctl = c.collect_sysctl()
+      diaglogger_info("[Collect] sysctl information is stored in #{sysctl}")
+			
+      diaglogger_info("[Valid] Validating systctl information...")
+      ret, sysctl = v.valid_sysctl(sysctl)
+      list =  sysctl.keys
+      list.each do |k|
+        if sysctl[k]['result'] == 'correct'
+          diaglogger_info("[Valid]    Sysctl: #{k} => #{sysctl[k]['value']} is correct (recommendation is #{sysctl[k]['recommend']})")
+        elsif sysctl[k]['result'] == 'incorrect'
+          diaglogger_warn("[Valid]    Sysctl: #{k} => #{sysctl[k]['value']} is incorrect (recommendation is #{sysctl[k]['recommend']})")
+        end
+      end
+
+      diaglogger_info("[Collect] Collecting ulimit information...")
+      ulimit = c.collect_ulimit()
+      diaglogger_info("[Collect] ulimit information is stored in #{ulimit}")
+
+      diaglogger_info("[Valid] Validating ulimit information...")
+      ret, rec, val = v.valid_ulimit(ulimit)
+      if ret == true
+        diaglogger_info("[Valid]    ulimit => #{val} is correct (recommendation is >#{rec})")
+      else
+        diaglogger_warn("[Valid]    ulimit => #{val} is incorrect (recommendation is >#{rec})")
+      end
+
+      if @conf[:mask] == 'yes'
+        diaglogger_info("[Mask] Masking td-agent config file : #{tdconf}...")
+        m.mask_tdlog(tdconf, clean = true)
+        tdlog.each do | file |
+          diaglogger_info("[Mask] Masking td-agent log file : #{file}...")
+          filename = file.split("/")[-1]
+          if filename.include?(".gz")
+            m.mask_tdlog_gz(file, clean = true)
+          elsif
+            m.mask_tdlog(file, clean = true)
+          end
+        end
+      end
+      
+      if @conf[:mask] == 'yes'
+        diaglogger_info("[Mask] Export mask log file : #{@masklog}")
+        m.export_masklog(@masklog)
+      end
+      
+      tar_file = c.compress_output()
+      diaglogger_info("[Collect] Generate tar file #{tar_file}")
+    end
+
+    def parse_diagconf(params)
+      options = {
+        :output_dir => '',
+        :mask => 'no',
+        :words => [],
+        :wfile => '',
+        :seed => ''
+      }
+      if params[:output] != nil
+        if Dir.exist?(params[:output])
+          options[:output_dir] = params[:output]
+        else
+          raise "output directory '#{output_dir}' does not exist"
+        end
+      else
+        raise "output directory '-o' must be specified"
+      end
+      if params[:mask] == nil
+        options[:mask] = 'no'
+      else
+        if params[:mask] == 'yes' || params[:mask] == 'no'
+          options[:mask] = params[:mask]
+        else
+          raise "invalid arguments '#{params[:mask]}' : input of '-m|--mask' should be 'yes' or 'no'"
+        end
+      end
+      options[:words] = params[:"word-list"] if params[:"word-list"] != nil
+      if params[:"word-file"] != nil
+        f = params[:"word-file"]
+        if File.exist?(f)
+          File.readlines(f).each do  |l|
+            options[:words].append(l.gsub(/\n/,''))
+          end
+        else
+          raise "#{params[:"word-file"]} : No such file or directory"
+        end
+      end
+      options[:words] = options[:words].uniq 
+      options[:seed] = params[:"hash-seed"] if params[:"hash-seed"] != nil
+      return options	
+    end
+    def diaglogger_debug(str)
+      @logger.debug(str)
+      @logger_file.debug(str)
+    end
+    def diaglogger_info(str)
+      @logger.info(str)
+      @logger_file.info(str)
+    end
+    def diaglogger_warn(str)
+      @logger.warn(str)
+      @logger_file.warn(str)
+    end
+    def diaglogger_error(str)
+      @logger.error(str)
+      @logger_file.error(str)
+    end
+  end
+end

data/lib/fluent/diagtool/maskutils.rb

@@ -0,0 +1,314 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'digest'
+require 'fileutils'
+require 'logger'
+require 'open3'
+require 'json'
+
+module Diagtool
+  class MaskUtils
+    def initialize(conf, log_level)
+      @words = conf[:words]
+      @logger = Logger.new(STDOUT, level: log_level, formatter: proc {|severity, datetime, progname, msg|
+        "#{datetime}: [Maskutils] [#{severity}] #{msg}\n"
+      })
+      @logger.debug("Initialize Maskutils: sanitized word = #{conf[:words]}")
+      @hash_seed = conf[:seed]
+      @id = {
+        :fid =>'',
+        :lid =>'',
+        :cid =>''
+      }
+      @masklog = Hash.new { |h,k| h[k] = Hash.new(&h.default_proc) }
+    end
+    def mask_tdlog(input_file, clean)
+      line_id = 0
+      f = File.open(input_file+'.mask', 'w')
+      File.readlines(input_file).each do |line|
+        line = line.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')   # temporary
+        @id[:fid] = input_file
+        @id[:lid] = line_id
+        line_masked = mask_tdlog_inspector(line)
+        f.puts(line_masked)
+        line_id+=1
+      end
+      f.close
+      FileUtils.rm(input_file) if clean == true
+      return input_file+'.mask'
+    end
+    def mask_tdlog_gz(input_file, clean)
+      line_id = 0
+      f = File.open(input_file+'.mask', 'w')
+      gunzip_file = input_file+'.mask'+'.tmp'
+      Open3.capture3("gunzip --keep -c #{input_file} > #{gunzip_file}")
+      File.readlines(gunzip_file).each do |line|
+        @id[:fid] = input_file
+        @id[:lid] = line_id
+        line_masked = mask_tdlog_inspector(line)
+        f.puts(line_masked)
+        line_id+=1
+      end
+      f.close
+      FileUtils.rm(gunzip_file)
+      FileUtils.rm(input_file) if clean == true
+      return input_file+'.mask'
+    end
+    def mask_tdlog_inspector(line)
+      i = 0
+      contents=[]
+      @logger.debug("Input Line: #{line.chomp}")
+      @logger.debug("Splitted Line: #{line.split(/\s/)}")
+      loop do
+        contents[i] = line.split(/\s/)[i].to_s
+        @logger.debug("Splitted Line #{i}: #{contents[i]}")
+        @id[:cid] = i.to_s
+        if contents[i].include?(',')
+          contents_s = contents[i].split(',')
+          cnt = 0
+          loop do
+            @id[:cid] = i.to_s + '-' + cnt.to_s
+            if contents_s[cnt].include?('://') ## Mask <http/dRuby>://<address:ip/hostname>:<port>
+              is_mask, masked_contents = mask_url_pattern(contents_s[cnt])
+              if is_mask
+                @logger.debug("   URL Pattern Detected: #{contents_s[cnt]} -> #{masked_contents}")
+                contents_s[cnt] = masked_contents
+              end
+            elsif contents_s[cnt].include?('=')
+              is_mask, masked_contents = mask_equal_pattern(contents_s[cnt])
+              if is_mask
+                @logger.debug("   Equal Pattern Detected: #{contents_s[cnt]} -> #{masked_contents}")
+                contents_s[cnt] = masked_contents
+              end
+            elsif contents_s[cnt].include?(':') ## Mask <address:ip/hostname>:<port>
+              is_mask, masked_contents = mask_colon_pattern(contents_s[cnt])
+              if is_mask
+                @logger.debug("   Colon Pattern Detected: #{contents_s[cnt]} -> #{masked_contents}")
+                contents_s[cnt] = masked_contents
+              end
+            elsif contents_s[cnt].include?('/') ## Mask <address:ip/hostname>:<port>
+              is_mask, masked_contents = mask_slash_pattern(contents_s[cnt])
+              if is_mask
+                @logger.debug("   Slash Pattern Detected: #{contents_s[cnt]} -> #{masked_contents}")
+                contents_s[cnt] = masked_contents
+              end
+            else 
+              is_mask, masked_contents = mask_direct_pattern(contents_s[cnt])
+              if is_mask
+                @logger.debug("   Direct Pattern Detected: #{contents_s[cnt]} -> #{masked_contents}")
+                contents_s[cnt] = masked_contents
+              end
+            end
+            cnt+=1
+            break if cnt >= contents_s.length 
+          end
+          contents[i] = contents_s.join(',')
+        else
+          if contents[i].include?('://') ## Mask <http/dRuby>://<address:ip/hostname>:<port>
+            is_mask, masked_contents = mask_url_pattern(contents[i])
+            if is_mask
+              @logger.debug("   URL Pattern Detected: #{contents[i]} -> #{masked_contents}")
+              contents[i] = masked_contents
+            end
+          elsif contents[i].include?('=')
+            is_mask, masked_contents = mask_equal_pattern(contents[i])
+            if is_mask
+              @logger.debug("   Equal Pattern Detected: #{contents[i]} -> #{masked_contents}")
+              contents[i] = masked_contents
+            end
+          elsif contents[i].include?(':') ## Mask <address:ip/hostname>:<port>
+            is_mask, masked_contents = mask_colon_pattern(contents[i])
+            if is_mask
+              @logger.debug("   Colon Pattern Detected: #{contents[i]} -> #{masked_contents}")
+              contents[i] = masked_contents
+            end
+          elsif contents[i].include?('/')
+            is_mask, masked_contents = mask_slash_pattern(contents[i])
+            if is_mask
+              @logger.debug("   Slash Pattern Detected: #{contents[i]} -> #{masked_contents}")
+              contents[i] = masked_contents
+            end
+          else
+            is_mask, masked_contents = mask_direct_pattern(contents[i])
+            if is_mask
+              @logger.debug("   Direct Pattern Detected: #{contents[i]} -> #{masked_contents}")
+              contents[i] = masked_contents
+            end
+          end
+        end
+        i+=1
+        break if i >= line.split(/\,|\s/).length
+      end
+      line_masked = contents.join(' ')
+      @logger.debug("Masked Line: #{line_masked}")
+      return line_masked
+    end
+    def mask_direct_pattern(str)
+      is_mask = false
+      if str.include?(">")
+        str = str.gsub(">",'')
+        is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(str)
+        str_m = chunk_mask + ">" if is_mask
+      elsif str.include?("]")
+        str = str.gsub("]",'')
+        is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(str)
+        str_m = chunk_mask + "]" if is_mask
+      else
+        is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(str)
+        str_mask = chunk_mask if is_mask
+      end
+      return is_mask, str_mask
+    end	
+    def mask_url_pattern(str)
+      is_mask = false
+      url = str.split('://')
+      cnt_url = 0
+      loop do
+        if url[cnt_url].include?(':')
+          address = url[cnt_url].split(':')
+          cnt_address = 0
+          loop do
+            if address[cnt_address].include?("]")
+              is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(address[cnt_address].gsub(']',''))
+              address[cnt_address] = chunk_mask + "]" if is_mask
+            elsif address[cnt_address].include?(">")
+              is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(address[cnt_address].gsub('>',''))
+              address[cnt_address] = chunk_mask + ">" if is_mask
+            else
+              is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(address[cnt_address])
+              address[cnt_address] = chunk_mask if is_mask
+            end
+            cnt_address+=1
+            break if cnt_address >= address.length || is_mask == true
+          end
+          url[cnt_url] = address.join(':')
+        else
+          if url[cnt_url].include?("]")
+            is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(url[cnt_url].gsub(']',''))
+            url[cnt_url] = chunk_mask + "]" if is_mask
+          elsif url[cnt_url].include?(">")
+            is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(url[cnt_url].gsub('>',''))
+            url[cnt_url] = chunk_mask + ">" if is_mask
+          else
+            is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(url[cnt_url])
+            url[cnt_url] = chunk_mask if is_mask
+          end
+        end
+        cnt_url+=1
+        break if cnt_url >= url.length || is_mask == true
+      end
+      str_mask = url.join('://')
+      str_mask << ":" if str.end_with?(':')
+      return is_mask, str_mask
+    end
+    def mask_equal_pattern(str)
+      is_mask = false
+      l = str.split('=') ## Mask host=<address:ip/hostname> or bind=<address: ip/hostname>
+      i = 0
+      loop do
+        is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(l[i])
+        l[i] = chunk_mask if is_mask
+        i+=1
+        break if i >= l.length || is_mask == true
+      end
+      str_mask = l.join('=')
+      return is_mask, str_mask
+    end
+    def mask_colon_pattern(str)
+      is_mask = false
+      l = str.split(':')
+      i = 0
+      loop do
+        is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(l[i])
+        l[i] = chunk_mask if is_mask
+        i+=1
+        break if i >= l.length || is_mask == true
+      end
+      str_mask = l.join(':')
+      str_mask << ":" if str.end_with?(':')
+      return is_mask, str_mask
+    end
+    def mask_slash_pattern(str)
+      is_mask = false
+      l = str.split('/')
+      i = 0
+      loop do
+        is_mask, chunk, chunk_mask = mask_ipv4_fqdn_words(l[i])
+        l[i] = chunk_mask if is_mask
+        i+=1
+        break if i >= l.length || is_mask == true
+      end
+      str_mask = l.join('/')
+      str_mask << ":" if str.end_with?(':')
+      return is_mask, str_mask
+    end
+    def is_ipv4?(str)
+      !!(str =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)
+    end
+    def is_fqdn?(str)
+      #!!(str =~ /^\b((?=[a-z0-9-]{1,63}\.)[a-z0-9]+(-[a-z0-9]+)*\.)+([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/)
+      !!(str =~ /^\b(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.){2,}([A-Za-z]|[A-Za-z][A-Za-z\-]*[A-Za-z]){2,}$/)
+      #!!(str =~ /^\b(?=^.{1,254}$)(^(?:(?!\d+\.)[a-zA-Z0-9_\-]{1,63}\.?)+(?:[a-zA-Z]{2,})$)/)
+    end
+    def is_words?(str)
+      value = false
+      @words.each do | l |
+        if str == l
+          value = true
+          break
+        end
+      end
+      return value
+    end
+    def mask_ipv4_fqdn_words(str)
+      str = str.to_s
+      mtype = ''
+      is_mask = false
+      if is_ipv4?(str.gsub(/\\\"|\'|\"|\\\'/,''))
+        str = str.gsub(/\\\"|\'|\"|\\\'/,'')
+        mtype = 'IPv4'
+        is_mask = true
+      elsif is_fqdn?(str.gsub(/\\\"|\'|\"|\\\'/,''))
+        str = str.gsub(/\\\"|\'|\"|\\\'/,'')
+        mtype = 'FQDN'
+        is_mask = true
+      elsif is_words?(str.gsub(/\\\"|\'|\"|\\\'/,''))
+        str = str.gsub(/\\\"|\'|\"|\\\'/,'')
+        mtype = 'Word'
+        is_mask =true
+      end
+      if is_mask
+        str_mask = mtype + '_' + Digest::MD5.hexdigest(@hash_seed + str)
+        put_masklog(str, str_mask)
+      else
+        str_mask = str
+      end
+      return is_mask, str, str_mask
+    end
+    def put_masklog(str, str_mask)
+      uid = "Line#{@id[:lid]}-#{@id[:cid]}"
+      @masklog[@id[:fid]][uid]['original'] = str
+      @masklog[@id[:fid]][uid]['mask'] = str_mask
+    end
+    def export_masklog(output_file)
+      masklog_json = JSON.pretty_generate(@masklog)
+      File.open(output_file, 'w') do |f|
+        f.puts(masklog_json)
+      end
+    end 
+  end
+end