fuzzr 0.9.9

data/lib/fuzz/fuzz.rb

@@ -0,0 +1,399 @@
+# encoding: utf-8
+# -------------------------------------------------------------------
+# fuzz.rb - TAOX11 fuzz checker
+#
+# Author: Martin Corino
+#
+# Copyright (c) Remedy IT Expertise BV
+# -------------------------------------------------------------------
+
+require 'optparse'
+require 'tempfile'
+require 'fileutils'
+require 'yaml'
+require 'fuzz/log'
+require 'fuzz/system'
+require 'fuzz/options'
+require 'fuzz/fzzr'
+require 'fuzz/version'
+
+module Fuzz
+
+  def self.root_path
+    f = File.expand_path(__FILE__)
+    f = File.expand_path(File.readlink(f)) if File.symlink?(f)
+    File.dirname(f)
+  end
+
+  FUZZ_ROOT = self.root_path
+
+  class << self
+
+    include LogMethods
+    include Sys::SysMethods
+
+    def reporter
+      @reporter ||= Fuzz::Reporter.new
+    end
+
+    def set_reporter(rep)
+      @reporter = rep
+    end
+    alias :reporter= :set_reporter
+
+    def options
+      Fuzz::OPTIONS
+    end
+
+    def reset
+      options.reset
+    end
+
+    def load_config
+      options.load_config
+    end
+
+    def includes
+      unless @include_re
+        @include_re = []
+        # @include_re << "\\.(#{Fuzz::FileObject.extensions.join('|')})$"
+        # @include_re << "#{Fuzz::FileObject.filenames.join('|')}$"
+        @include_re << "\\.(#{options.config[:exts].join('|')})$"
+        @include_re << "#{options.config[:filenames].join('|')}$"
+      end
+      @include_re
+    end
+
+    #
+    # fuzzer registration
+    #
+
+    def fuzzers
+      @fuzzers ||= {}
+    end
+
+    def register_fzzr(fzzr)
+      raise "Duplicate fuzzer registration: #{fzzr.fuzz_id}" if fuzzers.has_key?(fzzr.fuzz_id)
+      fuzzers[fzzr.fuzz_id] = fzzr
+      fzzr.setup(options.optparser) if options.optparser && fzzr.respond_to?(:setup) && fzzr_included?(fzzr)
+    end
+
+    def get_fzzr(id)
+      fuzzers[id]
+    end
+
+    def fzzr_excluded?(fzzr)
+      options.config[:fzzr_excludes].include?(fzzr.fuzz_id.to_sym)
+    end
+
+    def fzzr_included?(fzzr)
+      !fzzr_excluded?(fzzr)
+    end
+
+    #
+    # load fuzzers
+    #
+    def load_fuzzers
+      # standard fuzzers included in Gem
+      unless loaded_fzzr_paths.include?(_p = File.join(FUZZ_ROOT, 'fuzzers'))
+        Dir.glob(File.join(_p, '*.rb')).each do |fnm|
+          require fnm
+        end
+        loaded_fzzr_paths << _p
+      end
+      # configured fuzzers
+      options.config[:fzzr_paths].each do |fzzrpath|
+        unless loaded_fzzr_paths.include?(_p = File.expand_path(fzzrpath))
+          Dir.glob(File.join(_p, '*.rb')).each do |fnm|
+            require fnm
+          end
+          loaded_fzzr_paths << _p
+        end
+      end
+    end
+
+    private
+
+    def loaded_fzzr_paths
+      @loaded_fuzzr_paths ||= []
+    end
+
+  end
+
+  ## Backwards compatibility
+  def self.log_verbose(msg)
+    log_info(msg) if verbose?
+  end
+
+  #
+  # Option methods
+  #
+
+  def self.verbosity
+    options.verbose
+  end
+
+  def self.apply_fix?
+    options.apply_fix || false
+  end
+
+  def self.follow_symlink?
+    options.config[:follow_symlink] || false
+  end
+
+  def self.excludes
+    options.config[:excludes] || []
+  end
+
+  def self.excluded?(object)
+    excludes.any? { |excl| (object.fullpath =~ /#{excl}/) }
+  end
+
+  #
+  # parse commandline arguments
+  #
+  def self.init_optparser
+    script_name = File.basename($0)
+    if not script_name =~ /fuzz/
+      script_name = "ruby "+$0
+    end
+
+    options.optparser = opts = OptionParser.new
+    opts.banner = "Usage: #{script_name} [options] [glob [glob]]\n\n"
+    opts.separator "\n--- [General options] ---\n\n"
+    opts.on('-t', '--filetype', '=EXT', String,
+            'Defines an alternative filetype to search and scan. Can be specified multiple times.',
+            "Default: #{Fuzz::FileObject.extensions.join('|')}") { |v|
+              (options.user_config[:exts] ||= []) << v
+            }
+    opts.on('-f', '--file', '=NAME', String,
+            'Defines an alternative filename to search and scan. Can be specified multiple times.',
+            "Default: #{Fuzz::FileObject.filenames.join('|')}") { |v|
+              (options.user_config[:filenames] ||= []) << v
+            }
+    opts.on('-a', '--add-files',
+            'Add custom filenames and/or filetype extensions to default list instead of replacing defaults.',
+            'Default: false') { |v|
+              options.user_config[:add_files] = true
+            }
+    opts.on('-S', '--no-symlinks',
+            'Do not follow symlinks.',
+            'Default: follow symlinks') { |v|
+              options.user_config[:follow_symlink] = false
+            }
+    opts.on('-P', '--fzzr-path', '=PATH',
+            'Adds search path for Fuzzers.',
+            "Default: loaded from ~/#{FUZZRC} and/or ./#{FUZZRC}") { |v|
+              (options.user_config[:fzzr_paths] ||= []) << v.to_s
+            }
+    opts.on('-B', '--blacklist', '=FZZRID',
+            'Adds Fuzzer ID to list of fuzzers to exclude from Fuzz check.',
+            'Default: none') { |v|
+              (options.user_config[:fzzr_excludes] ||= []) << v.to_sym
+            }
+    opts.on('-X', '--exclude', '=MASK',
+            'Adds path mask (regular expression) to list to exclude from Fuzz check.',
+            'Default: none') { |v|
+              (options.user_config[:excludes] ||= []) << v
+            }
+    opts.on('-c', '--config', '=FUZZRC',
+            'Load config from FUZZRC file.',
+            "Default:  ~/#{FUZZRC} and/or ./#{FUZZRC}") { |v|
+      options.add_config(v)
+    }
+    opts.on('--write-config', '=[FUZZRC]',
+            'Write config to file and exit.',
+            "Default: ./#{FUZZRC}") { |v|
+      options.user_config.save(String === v ? v : FUZZRC)
+      exit
+    }
+    opts.on('--show-config',
+            'Display config settings and exit.') { |v|
+      options.load_config
+      puts YAML.dump(options.config.__send__ :table)
+      exit
+    }
+
+    opts.separator ''
+    opts.on('-o', '--output', '=FILE', String,
+            'Specifies filename to write Fuzz messages to.',
+            'Default: stderr') { |v|
+              options.output = v
+            }
+    opts.on('-p', '--apply-fix',
+            'Apply fixes (if any) for Fuzz errors.',
+            'Default: false') { |v|
+              options.apply_fix = true
+            }
+    opts.on('-n', '--no-recurse',
+            'Prevents directory recursion in file selection.',
+            'Default: recurse') { |v|
+              options.recurse = false
+            }
+    opts.on('-v', '--verbose',
+            'Run with increased verbosity level. Repeat to increase more.',
+            'Default: 1') { |v| options.verbose += 1 }
+
+    opts.separator ''
+    opts.on('-L', '--list',
+            'List available Fuzzers and exit.') {
+      options.load_config
+      load_fuzzers
+      puts "TAOX11 fuzz checker #{FUZZ_VERSION_MAJOR}.#{FUZZ_VERSION_MINOR}.#{FUZZ_VERSION_RELEASE}"
+      puts FUZZ_COPYRIGHT
+      puts('%-30s %s' % %w{Fuzzer Description})
+      puts(('-' * 30)+' '+('-' * 48))
+      fuzzers.values.each { |fzzr| puts('%-30s %s' % [fzzr.fuzz_id, fzzr.description]) }
+      puts
+      exit
+    }
+
+    opts.separator ""
+    opts.on('-V', '--version',
+            'Show version information and exit.') {
+      puts "TAOX11 fuzz checker #{FUZZ_VERSION_MAJOR}.#{FUZZ_VERSION_MINOR}.#{FUZZ_VERSION_RELEASE}"
+      puts FUZZ_COPYRIGHT
+      exit
+    }
+    opts.on('-h', '--help',
+            'Show this help message.') {
+      options.load_config
+      load_fuzzers
+      puts opts;
+      puts;
+      exit
+    }
+
+    opts.separator "\n--- [Fuzzer options] ---\n\n"
+  end
+
+  def self.parse_args(argv)
+    options.optparser.parse!(argv)
+  end
+
+  def self.select_fzzrs(object)
+    fuzzers.values.collect { |fzzr| (fzzr_included?(fzzr) && fzzr.applies_to?(object)) ? fzzr : nil }.compact
+  end
+
+  def self.update_file_object(fo)
+    if File.writable?(fo.fullpath)
+      log_verbose(%Q{Updating #{fo}...})
+      ftmp = Tempfile.new(fo.name)
+      log_verbose(%Q{+ Writing temp file #{ftmp.path}...})
+      fo.lines.each { |ln| ftmp.print ln }
+      ftmp.close(false) # close but do NOT unlink
+      log_verbose(%Q{+ Replacing #{fo} with #{ftmp.path}})
+      # create temporary backup
+      ftmp2 = Tempfile.new(fo.name)
+      ftmp2_name = ftmp2.path.dup
+      ftmp2.close(true)
+      mv(fo.fullpath, ftmp2_name)
+      # replace original
+      begin
+        mv(ftmp.path, fo.fullpath)
+        # preserve file mode
+        chmod(File.lstat(ftmp2_name).mode, fo.fullpath)
+      rescue
+        log_error(%Q{FAILED updating #{fo}: #{$!}})
+        # restore backup
+        mv(ftmp2_name, fo.fullpath)
+        raise
+      end
+      # remove backup
+      File.unlink(ftmp2_name)
+      log_verbose(%Q{Finished updating #{fo}.})
+      return true
+    else
+      log_error(%Q{NO_ACCESS - cannot update #{fo}})
+      return false
+    end
+  end
+
+  def self.handle_object(object)
+    log_verbose(%Q{Handling #{object}})
+    fzzrs = select_fzzrs(object)
+    no_fixes_allowed = false
+    rc = fzzrs.inject(true) do |result, fzzr|
+      log_verbose(%Q{+ Running fuzzer #{fzzr.fuzz_id}})
+      begin
+        if fzzr.run(object, options.apply_fix)
+          result
+        else
+          log_verbose(%Q{+ Error from fuzzer #{fzzr.fuzz_id}})
+          false
+        end
+      rescue
+        log_error(%Q{EXCEPTION CAUGHT running fuzzer #{fzzr.fuzz_id} on #{object} - #{$!}\n#{$!.backtrace.join("\n")}})
+        no_fixes_allowed = true
+        break ## immediately stop handling this object, rc will remain false
+      end
+    end
+    unless no_fixes_allowed
+      if Fuzz.apply_fix? && object.changed?
+        rc = update_file_object(object) && rc
+      end
+    end
+    rc ? true : false
+  end
+
+  def self.iterate_paths(paths)
+    paths.inject(true) do |result, path|
+      if File.readable?(path) && (!File.symlink?(path) || follow_symlink?)
+        if File.directory?(path)
+          rc = handle_object(dirobj = Fuzz::DirObject.new(path))
+          log_verbose(%Q{Iterating #{path}})
+          if options.recurse && !excluded?(dirobj)
+            rc = iterate_paths(Dir.glob(File.join(path, '*'))) && rc
+          end
+          rc
+        elsif File.file?(path)
+          handle_object(Fuzz::FileObject.new(path))
+        else
+          true
+        end
+      else
+        log_warning(File.readable?(path) ? %Q{Cannot read #{path}} : %Q{Cannot follow symlink #{path}})
+        false
+      end && result
+    end
+  end
+
+  def self.run_fzzrs(argv)
+    options.config[:exts].concat(Fuzz::FileObject.extensions) if options.config[:exts].empty? || options.config[:add_files]
+    options.config[:filenames].concat(Fuzz::FileObject.filenames) if options.config[:filenames].empty? || options.config[:add_files]
+
+    options.config[:exts].uniq!
+    options.config[:filenames].uniq!
+
+    f_close_output = false
+    if String === options.output
+      options.output = File.open(options.output, 'w')
+      f_close_output = true
+    end
+    begin
+      # determin files/paths to test
+      paths = argv.collect { |a| Dir.glob(a) }.flatten.uniq
+      paths = Dir.glob('*') if paths.empty?
+      # scan all determined objects
+      return iterate_paths(paths)
+    ensure
+      options.output.close if f_close_output
+    end
+  end
+
+  def self.run
+    init_optparser
+
+    # parse arguments
+    parse_args(ARGV)
+
+    # load config (if any)
+    options.load_config
+
+    # load fuzzers
+    load_fuzzers
+
+    run_fzzrs(ARGV)
+  end
+
+end

data/lib/fuzz/fuzzers/check_whitespace.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+# -------------------------------------------------------------------
+# check_whitespace.rb - TAOX11 whitespace checker
+#
+# Author: Martin Corino
+#
+# Copyright (c) Remedy IT Expertise BV
+# -------------------------------------------------------------------
+
+module Fuzzers
+  class WhitespaceChecker
+    include Fuzz::Fzzr
+    def initialize
+      @fuzz_id = :check_whitespace
+      @description = 'checks for trailing whitespace, incorrect line endings and tabs'
+    end
+
+    def setup(optparser)
+      optparser.on('--wsc:tab-spacing=NUM', Integer,
+                   'Fuzzers::WhitespaceChecker - defines tab spacing to use for TAB replacement when --apply-fix is enabled.',
+                   "Default: #{tab_spacing}") {|v| self.options[:tabspacing] = v }
+    end
+
+    def applies_to?(object)
+      Fuzz::FileObject === object && !is_excluded?(object)
+    end
+
+    def run(object, apply_fix)
+      _tws = []
+      _tabs = []
+      object.iterate(fuzz_id) do |lnptr|
+        if lnptr.text =~ /(\s\n|[\ \t\f\r\x0B])\Z/
+          if apply_fix
+            Fuzz.log_verbose(%Q{#{object.path}:#{lnptr.line_nr} - stripping trailing whitespace})
+            lnptr.text.rstrip!
+            lnptr.text << "\n" if $1.end_with?("\n")
+          else
+            _tws << lnptr.line_nr
+          end
+        end
+        if lnptr.text =~ /\t/
+          if apply_fix
+            Fuzz.log_warning(%Q{#{object.path}:#{lnptr.line_nr} - replacing tabs})
+            lnptr.text.gsub!(/\t/, ' ' * tab_spacing)
+          else
+            _tabs << lnptr.line_nr
+          end
+        end
+      end
+      Fuzz.log_error(%Q{#{object.path}:[#{_tws.join(',')}] trailing whitespace or incorrect line ending detected}) unless _tws.empty?
+      Fuzz.log_error(%Q{#{object.path}:[#{_tabs.join(',')}] tab(s) detected}) unless _tabs.empty?
+      return (_tws.empty? && _tabs.empty?)
+    end
+
+  private
+    def tab_spacing
+      self.options[:tabspacing] || 2
+    end
+  end
+
+  Fuzz.register_fzzr(WhitespaceChecker.new)
+end

data/lib/fuzz/fzzr.rb

@@ -0,0 +1,258 @@
+# encoding: utf-8
+# -------------------------------------------------------------------
+# fuzzr.rb - TAOX11 Fuzzer bases
+#
+# Author: Martin Corino
+#
+# Copyright (c) Remedy IT Expertise BV
+# -------------------------------------------------------------------
+
+module Fuzz
+  ##
+  # Fuzzers are objects having the following readonly attributes:
+  #   #fuzz_id    : id of fuzzer (Symbol)
+  #   #description: (String)
+  #   #errormsg   : (String)
+  #
+  # and having the following methods:
+  #   #applies_to?(object)  : checks if the test applies to the object passed
+  #                           (Fuzz::DirObject or Fuzz::FileObject)
+  #   #run(object,apply_fix): runs the fzzr test on the object passed
+  #                           (Fuzz::DirObject or Fuzz::FileObject)
+  #                           when apply_fix == true Fuzzer is directed to
+  #                           attempt to fix any problems found (future)
+  #
+  # Fuzz::Fzzr is provided as a convenience Mixin for fuzzers
+  #
+  # Fuzzers can inspect the options passed to fuzz.rb by referencing Fuzz::OPTIONS
+  ##
+
+  module Fzzr
+    attr_reader :fuzz_id, :description, :errormsg
+
+    def applies_to?(object)
+      !is_excluded?(object)
+    end
+
+    def setup(optparser)
+    end
+
+    def run(object, apply_fix)
+      true
+    end
+
+    def is_excluded?(object)
+      # force excludes to be parsed
+      _excludes
+      # now examine
+      ((!_is_included?(object)) || _excludes.any? { |excl| (object.fullpath =~ /#{excl}/) })
+    end
+
+    def options
+      Fuzz::OPTIONS[:config][:fzzr_opts][self.fuzz_id] ||= {}
+    end
+
+    private
+
+    def _is_included?(object)
+      Fuzz::DirObject === object || _includes.empty? || _includes.any? { |incl| (object.fullpath =~ /#{incl}/) }
+    end
+
+    def _includes
+      @_includes ||= Fuzz.includes.dup
+    end
+
+    def _excludes
+      unless @_excludes
+        @_excludes = []
+        @_excludes.concat(Fuzz.excludes)
+        Fuzz::OPTIONS[:config][:fzzr_paths].each do |fzzrpath|
+          fzzr_excl_file = File.join(fzzrpath, "#{self.fuzz_id}.excludes")
+          if File.readable?(fzzr_excl_file)
+            lns = IO.readlines(fzzr_excl_file).collect { |l| l.strip }
+            @_excludes.concat(lns.select {|l| !(l.empty? || l[0] == '!') })
+            _includes.concat(lns.select {|l| !(l.empty? || l[0] != '!') }.collect {|l| l[1,l.size].strip })
+          else
+            false
+          end
+        end
+      end
+      @_excludes
+    end
+  end # Fzzr
+
+  class DirObject
+    attr_reader :path, :fullpath, :name, :ext
+    def initialize(path)
+      @path = path
+      @fullpath = File.expand_path(path)
+      @name = File.basename(path)
+      @ext = File.extname(path).sub(/^\./,'')
+    end
+
+    def changed?
+      false
+    end
+
+    def iterate(fzzr_id, &block)
+      # nothing to iterate over
+      true
+    end
+
+    def to_s
+      "Dir:#{path}"
+    end
+  end # DirObject
+
+  class FileObject
+    EXTS = [
+      'h', 'hxx', 'hpp', 'c', 'cc', 'cxx', 'cpp', 'H', 'C', 'inl', 'asm',
+      'rb', 'erb', 'pl', 'pm', 'py',
+      'idl', 'pidl',
+      'mwc', 'mpc', 'mpb', 'mpt', 'mpd',
+      'cdp', 'xml', 'conf', 'html',
+      'asc', 'adoc'
+      ]
+    FILES = [
+      'ChangeLog', 'README'
+      ]
+
+    def self.extensions
+      EXTS
+    end
+
+    def self.filenames
+      FILES
+    end
+
+    class LinePointer
+      attr_reader :err_lines
+
+      FZZR_ENABLE_RE = /X11_FUZZ\: enable ([^\s]+)/
+      FZZR_DISABLE_RE = /X11_FUZZ\: disable ([^\s]+)/
+
+      def initialize(lines, fzzr_id)
+        @lines = lines
+        @fzzr_id = fzzr_id.to_s
+        @err_lines = []
+        reset
+      end
+      def fzzr_disabled?
+        @fzzr_disabled
+      end
+      def line_nr
+        @line_nr+1
+      end
+      def text_at(offs)
+        ln = @line_nr+offs
+        if ln>=0 && ln<@lines.size
+          return @lines[ln]
+        end
+        nil
+      end
+      def set_text_at(offs, txt)
+        ln = @line_nr+offs
+        if ln>=0 && ln<@lines.size
+          return (@lines[ln] = txt)
+        end
+        nil
+      end
+      def text
+        text_at(0)
+      end
+      def text=(txt)
+        set_text_at(0, txt)
+      end
+      def move(offs)
+        if offs < 0
+          _backward(-offs) unless bof?
+        else
+          _forward(offs) unless eof?
+        end
+        self.line_nr
+      end
+      def reset
+        @line_nr = 0
+        @fzzr_disabled = false
+        _check_fzzr_escape
+      end
+      def to_eof
+        _forward(@lines.size - @line_nr)
+      end
+      def eof?
+        @line_nr >= @lines.size
+      end
+      def bof?
+        @line_nr <= 0
+      end
+      def mark_error(ln = nil)
+        @err_lines << (ln || (@line_nr+1))
+      end
+
+      private
+
+      def _forward(distance)
+        distance.times do
+          @line_nr += 1
+          break if eof?
+          _check_fzzr_escape
+        end
+      end
+
+      def _backward(distance)
+        distance.times do
+          break if bof?
+          @line_nr -= 1
+          _check_fzzr_escape(false)
+        end
+      end
+
+      def _check_fzzr_escape(forward = true)
+        begin
+          if FZZR_ENABLE_RE =~ @lines[@line_nr]
+            @fzzr_disabled = !forward if $1 == @fzzr_id
+          elsif FZZR_DISABLE_RE =~ @lines[@line_nr]
+            @fzzr_disabled = forward if $1 == @fzzr_id
+          end
+        rescue
+          Fuzz.log_error(%Q{ERROR: Exception while checking fzzr escapes in line #{@line_nr+1} - #{$!}\n#{@lines[@line_nr]}})
+          raise
+        end
+      end
+    end # LinePointer
+
+    attr_reader :path, :fullpath, :name, :ext, :lines
+
+    def initialize(path)
+      @path = path
+      @fullpath = File.expand_path(path)
+      @name = File.basename(path)
+      @ext = File.extname(path).sub(/^\./,'')
+      @lines = nil
+      @pointer = nil
+      @changed = false
+    end
+
+    def changed?
+      @changed
+    end
+
+    def iterate(fzzr_id, &block)
+      @lines ||= IO.readlines(fullpath)
+      lines_copy = @lines.collect {|l| l.dup }
+      pointer = LinePointer.new(@lines, fzzr_id)
+      begin
+        block.call(pointer) unless pointer.fzzr_disabled?
+        pointer.move(1)
+      end while !pointer.eof?
+      Fuzz.log_error(%Q{#{self.path}[#{pointer.err_lines.join(',')}] #{Fuzz.get_fzzr(fzzr_id).errormsg}}) unless pointer.err_lines.empty?
+      @changed |= (@lines != lines_copy)
+      lines_copy = nil
+      return pointer.err_lines.empty?
+    end
+
+    def to_s
+      "File:#{fullpath}"
+    end
+  end # FileObject
+end # Fuzz