rak 1.0 → 1.2

data/bin/rak

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
+# encoding: binary
 
 require 'rubygems'
+require "pathname"
 
 if RUBY_PLATFORM =~ /mswin|mingw/
   begin
@@ -20,8 +22,16 @@ rescue LoadError
   $use_onig = false
 end
 
+class String
+  def expand_tabs(shift=0)
+    expanded = dup
+    1 while expanded.sub!(/\t+/){ " "*($&.size*8 - ($`.size+shift)%8) }
+    expanded
+  end
+end
+
 class Rak
-  VERSION = "0.9"
+  VERSION = "1.2"
   
   FILE_COLOUR = "\033[1;31m"
   MATCH_COLOUR = "\033[1;37m\033[41m"
@@ -30,7 +40,7 @@ class Rak
   VERSION_INFO=<<END
 rak #{VERSION}
 
-Copyright 2008 Daniel Lucraft, all rights reserved. 
+Copyright 2008-#{Time.now.year} Daniel Lucraft, all rights reserved. 
 Based on the perl tool 'ack' by Andy Lester.
 
 This program is free software; you can redistribute it and/or modify it
@@ -38,40 +48,57 @@ under the same terms as Ruby.
 END
   
   FILE_TYPES = {
-    :asm         => %w( .s .S ),
-    :cc          => %w( .c .h .xs ),
-    :cpp         => %w( .cpp .cc .m .h .hpp .C .H ),
-    :csharp      => %w( .cs ),
-    :css         => %w( .css ),
-    :elisp       => %w( .el ),
-    :haskell     => %w( .hs .lhs ),
-    :hh          => %w( .h ),
-    :html        => %w( .htm .html .shtml ),
-    :lisp        => %w( .lisp ),
-    :java        => %w( .java properties ),
-    :js          => %w( .js ),
-    :jsp         => %w( .jsp .jspx .jhtm .jhtml ),
-    :make        => %w( Makefile ),
-    :mason       => %w( .mas .mhtml .mpl .mtxt ),
-    :ocaml       => %w( .ml .mli ),
-    :parrot      => %w( .pir .pasm .pmc .ops .pod .pg .tg ),
-    :perl        => %w( .pl .pm .pod .t ),
-    :php         => %w( .php .phpt .php3 .php4 .php5 ),
-    :prolog      => %w( .pl .ecl ),
-    :python      => %w( .py ),
-    :ruby        => %w( .rb .rhtml .rjs .rxml Rakefile .rake .erb .haml),
-    :scheme      => %w( .scm ),
-    :shell       => %w( .sh .bash .csh .ksh .zsh ),
-    :sql         => %w( .sql .ctl ),
-    :tcl         => %w( .tcl ),
-    :tex         => %w( .tex .cls .sty ),
-    :text        => %w( .txt .text ),
-    :tt          => %w( .tt .tt2 .ttml ),
-    :vala        => %w( .vala .vapi ),
-    :vb          => %w( .bas .cls .frm .ctl .vb .resx ),
-    :vim         => %w( .vim ),
-    :yaml        => %w( .yaml .yml ),
-    :xml         => %w( .xml .dtd .xslt )
+    :actionscript  => %w( .as .mxml ),
+    :ada           => %w( .ada .adb .ads ),
+    :asm           => %w( .S .asm .s ),
+    :awk           => %w( .awk ),
+    :batch         => %w( .bat .cmd ),
+    :cc            => %w( .c .h .l .xs .y ),
+    :cfmx          => %w( .cfc .cfm .cfml ),
+    :cpp           => %w( .C .H .cc .cpp .cxx .h .hh .hpp .hxx .m ),
+    :csharp        => %w( .cs ),
+    :css           => %w( .css ),
+    :elisp         => %w( .el ),
+    :erlang        => %w( .erl .hrl ),
+    :fortran       => %w( .f .f03 .f77 .f90 .f95 .for .fpp .ftn ),
+    :haskell       => %w( .hs .lhs ),
+    :hh            => %w( .h ),
+    :html          => %w( .htm .html .shtml .xhtml ),
+    :java          => %w( .java .properties properties ),
+    :js            => %w( .js ),
+    :jsp           => %w( .jhtm .jhtml .jsp .jspx ),
+    :lisp          => %w( .lisp .lsp ),
+    :lua           => %w( .lua ),
+    :make          => %w( .mk Makefile ),
+    :mason         => %w( .mas .mhtml .mpl .mtxt ),
+    :matlab        => %w( .m .oct ),
+    :objc          => %w( .h .m ),
+    :objcpp        => %w( .h .mm ),
+    :ocaml         => %w( .ml .mli .mll .mly ),
+    :parrot        => %w( .ops .pasm .pg .pir .pmc .pod .tg ),
+    :perl          => %w( .pl .pm .pod .t ),
+    :php           => %w( .php .php3 .php4 .php5 .phpt .phtml ),
+    :plone         => %w( .cpt .cpy .metadata .pt .py ),
+    :prolog        => %w( .ecl .pl ),
+    :python        => %w( .py ),
+    :ruby          => %w( .erb .haml .rake .rb .rhtml .rjs .rxml Rakefile ),
+    :scala         => %w( .scala ),
+    :scheme        => %w( .scm .ss ),
+    :sed           => %w( .sed ),
+    :shell         => %w( .bash .csh .ksh .sh .tcsh .zsh ),
+    :smalltalk     => %w( .st ),
+    :sml           => %w( .sml .cm .sig ),
+    :sql           => %w( .ctl .sql ),
+    :tcl           => %w( .itcl .itk .tcl ),
+    :tex           => %w( .cls .sty .tex ),
+    :text          => %w( .text .txt README ),
+    :tt            => %w( .tt .tt2 .ttml ),
+    :vala          => %w( .vala .vapi ),
+    :vb            => %w( .bas .cls .ctl .frm .resx .vb ),
+    :verilog       => %w( .v .verilog ),
+    :vim           => %w( .vim ),
+    :xml           => %w( .dtd .ent .xml .xslt ),
+    :yaml          => %w( .yaml .yml ),
   }
   
   VC_DIRS = %w(blib CVS _darcs .git .pc RCS SCCS .svn pkg)
@@ -80,6 +107,14 @@ END
     attr_reader :opt
   end
   
+  def self.compile_regexp(str)
+    if $use_onig
+      Oniguruma::ORegexp.new(str)
+    else
+      Regexp.new(str)
+    end
+  end
+  
   def self.search
     @opt = {}
 
@@ -104,6 +139,7 @@ END
                             [ '--help', GetoptLong::OPTIONAL_ARGUMENT ],
                             [ '--max-count', '-m', GetoptLong::REQUIRED_ARGUMENT ],
                             [ '--files', '-f', GetoptLong::NO_ARGUMENT ],
+                            [ '--skipped', GetoptLong::NO_ARGUMENT ],
                             [ '--output', GetoptLong::REQUIRED_ARGUMENT ],
                             [ '--version', GetoptLong::NO_ARGUMENT ],
                             [ '-c', '--count', GetoptLong::NO_ARGUMENT ],
@@ -136,16 +172,14 @@ END
                             [ '-k', GetoptLong::REQUIRED_ARGUMENT ],
                             [ '-x', '--line-regexp', GetoptLong::NO_ARGUMENT ],
                             [ '-s', '--line-start', GetoptLong::NO_ARGUMENT ],
-                            [ '-e', '--line-end', GetoptLong::NO_ARGUMENT ]
+                            [ '-e', '--line-end', GetoptLong::NO_ARGUMENT ],
+                            [ '--eval', GetoptLong::REQUIRED_ARGUMENT ]
                             )
 
-    dir = nil
-    
     opt[:max_count] = nil
-    opt[:do_search] = true
+    opt[:only_print_filelist] = false
     opt[:print_filename] = true
     opt[:print_line_number] = true
-    opt[:print_filelist] = false
     opt[:print_output] = nil
     opt[:print_highlighted] = true
     opt[:print_num_matches] = false
@@ -171,10 +205,12 @@ END
     opt[:collect_context] = false
     opt[:filename_regex] = nil
     opt[:neg_filename_regex] = nil
+    opt[:reverse_relevance] = false
+    opt[:eval] = nil
     
     # if redirected (RAK_TEST allows us to redirect in testing and still 
     # get the non-redirected defaults).
-    if STDOUT.isatty == false and ENV['RAK_TEST'] != "true"
+    unless STDOUT.isatty or ENV['RAK_TEST'] == "true"
       opt[:colour] = false
       opt[:print_file_each_line] = true
       opt[:print_filename] = false
@@ -189,13 +225,20 @@ END
             puts USAGE_HELP
           elsif arg == "types" or arg == "type"
             puts TYPES_HELP
+            FILE_TYPES.sort_by{|type,exts| type.to_s}.each do |type, exts|
+              puts "    --[no]%-13s %s\n" % [type, exts.join(" ")]
+            end
           end
           exit
+        when '--eval'
+          opt[:eval] = arg
         when '--max-count'
           opt[:max_count] = arg.to_i
         when '--files'
-          opt[:do_search] = false
-          opt[:print_filelist] = true
+          opt[:only_print_filelist] = true
+        when '--skipped'
+          opt[:reverse_relevance] = true
+          opt[:only_print_filelist] = true
         when '--output'
           opt[:print_filename] = false
           opt[:print_line_number] = false
@@ -291,9 +334,9 @@ END
           opt[:before_context] = val
           opt[:after_context] = val
         when '-g'
-          opt[:filename_regex] = arg
+          opt[:filename_regex] = compile_regexp(arg)
         when '-k'
-          opt[:neg_filename_regex] = arg
+          opt[:neg_filename_regex] = compile_regexp(arg)
         when '-x'
           opt[:match_whole_lines] = true
         when '-s'
@@ -309,230 +352,149 @@ END
       exit
     end
 
-    if ARGV.empty? and opt[:print_filelist] == false
-      puts USAGE_HELP
-      exit
-    end
-    
-    files = get_files
-    
-    compile_match_file
-
     unless opt[:colour]
       FILE_COLOUR.replace ""
       CLEAR_COLOURS.replace ""
       MATCH_COLOUR.replace ""
     end
 
-    print_files(files)      if opt[:print_filelist]
-    search2(ARGV[0], files) if opt[:do_search]
-  end
-  
-  def self.print_files(files)
-    files.each do |fn|
-      puts fn
+    if opt[:only_print_filelist]
+      each_file(ARGV) do |fn|
+        puts fn
+      end
+    elsif ARGV.empty? and !opt[:eval]
+      puts USAGE_HELP
+      exit
+    else
+      unless opt[:eval]
+        re = compile_pattern(ARGV.shift)
+      end
+      compiled = false
+      file_separator = ""
+      each_file(ARGV) do |fn|
+        # each_file might turn off printing file name, but only before first yield
+        unless compiled
+          compile_match_file
+          compiled = true
+        end
+        match_file(re, fn, file_separator)
+      end
     end
   end
   
-  def self.get_all_files(files=[], dir="./")
-    Dir[dir+"*"].each do |fn|
-      next if fn == ".."
-      if File.directory? fn
-        if opt[:descend] 
-          if File.symlink? fn
-            if opt[:follow_symlinks]
-              get_all_files(files, fn+"/")
-            end
-          else
-            get_all_files(files, fn+"/")
-          end
-        end
-      else
-        if !File.socket? fn and File.readable? fn
-          if File.symlink? fn
-            if opt[:follow_symlinks]
-              files << fn
-            end
-          else
-            files << fn
-          end
-        end
+  def self.extension_regexp(extensions)
+    return nil if extensions.empty?
+    Regexp.compile('(?:' + extensions.map{|x| Regexp.escape(x)}.join("|") + ')\z')
+  end
+
+  def self.file_relevant?(fn)
+    # These don't change at this point
+    @types_rx    ||= extension_regexp(FILE_TYPES.values.flatten)
+    @includes_rx ||= extension_regexp(opt[:includes])
+    @excludes_rx ||= extension_regexp(opt[:excludes])
+
+    ext = fn.basename.to_s
+    ext = shebang_matches(fn) unless ext =~ @types_rx
+
+    return false if !opt[:all_files]         and !ext or fn.to_s =~ /[~#]\z/
+    return false if @includes_rx             and (ext||"") !~ @includes_rx
+    return false if @excludes_rx             and (ext||"") =~ @excludes_rx
+    return false if opt[:filename_regex]     and fn.to_s  !~ opt[:filename_regex]
+    return false if opt[:neg_filename_regex] and fn.to_s  =~ opt[:neg_filename_regex]
+    return true
+  end
+
+  def self.find_all_files(path, &blk)
+    return if path.socket?
+    return unless path.readable?
+
+    if path.file?
+      relevant = file_relevant?(path)
+      relevant = !relevant if opt[:reverse_relevance]
+      yield(path.to_s.sub(/\A\.\/+/, "").gsub(/\/+/, "/")) if relevant
+    elsif path.directory?
+      path.children.each do |fn|
+        next if VC_DIRS.any?{|vc| vc == fn.basename.to_s}
+        next if fn.directory? and not opt[:descend]
+        next if fn.symlink? and not opt[:follow_symlinks]
+        find_all_files(fn, &blk)
       end
     end
-    files
   end
-  
-  def self.get_files(dir="./")
-    types_files = FILE_TYPES.invert
-    types = types_files.keys.flatten
-    files_and_dirs = ARGV[1..-1]
-    io_files = []
-    if !files_and_dirs or files_and_dirs.empty?
-      if STDIN.isatty == false
-        io_files << STDIN
-        opt[:print_filename] = false
-        files = []
+
+  def self.each_file(todo, &blk)
+    todo = todo.map{|path| Pathname(path)}
+    if todo.empty?
+      if STDIN.isatty
+        todo = [Pathname(".")]
       else
-        files = get_all_files
-      end
-    else
-      files = []
-      files_and_dirs.each do |thing|
-        if File.file? thing
-          files << thing
-        end
-        if File.directory? thing
-          get_all_files(files, thing)
-        end
-      end
-      if files_and_dirs.length == 1 and 
-          File.file? files_and_dirs[0]
         opt[:print_filename] = false
+        yield(STDIN)
+        return
       end
+    elsif todo.size == 1 and todo[0].file?
+      opt[:print_filename] = false
     end
     
-    # strip out based on extension
-    shebang_files = []
-    unless opt[:all_files]
-      files = files.select do |fn|
-        fn[-1..-1] != "~" and
-          fn[-1..-1] != "#"
-      end
-      new_files = []
-      files.each do |fn|
-        if types.any? {|ext| File.extname(fn) == ext or fn.split("/").last == ext}
-          new_files << fn
-        elsif ext = shebang_matches(fn)
-          shebang_files << [fn, ext]
-        end
-      end
-      files = new_files
-    end
-    
-    # select based on file type
-    if not opt[:includes].empty?
-      files = files.select do |fn|
-        opt[:includes].any? {|ext| fn.include? ext}
-      end
-      shebang_files2 = []
-      shebang_files.each do |fn, ext|
-        if opt[:includes].include? ext
-          shebang_files2 << [fn, ext]
+    if opt[:sort_files]
+      sortme = []
+      todo.each do |item|
+        find_all_files(item) do |fn|
+          sortme << fn
         end
       end
+      sortme.sort_by{|fn|fn.downcase}.each(&blk)
     else
-      shebang_files2 = shebang_files
-    end
-    
-    if not opt[:excludes].empty?
-      files = files.select do |fn|
-        not opt[:excludes].any? {|ext| fn.include? ext}
-      end
-      shebang_files3 = []
-      shebang_files2.each do |fn, ext|
-        unless opt[:excludes].any? {|ext| fn.include? ext}
-          shebang_files3 << [fn, ext]
-        end
+      todo.each do |item|
+        find_all_files(item, &blk)
       end
-    else
-      shebang_files3 = shebang_files2
-    end
-    
-    files += shebang_files3.map {|fn, ext| fn}
-    
-    # reject version control dirs
-    files.reject! do |fn|
-      VC_DIRS.any? {|vc| fn.include? vc}
     end
-    
-    # filter based on -g=REGEX
-    if opt[:filename_regex]
-      if $use_onig
-        fn_re = Oniguruma::ORegexp.new(opt[:filename_regex])
-      else
-        fn_re = Regexp.new(opt[:filename_regex])
-      end
-      files = files.select {|fn| fn =~ fn_re }
-    end
-    
-    # filter based on -k=REGEX
-    if opt[:neg_filename_regex]
-      if $use_onig
-        fn_re = Oniguruma::ORegexp.new(opt[:neg_filename_regex])
-      else
-        fn_re = Regexp.new(opt[:neg_filename_regex])
-      end
-      files = files.reject {|fn| fn =~ fn_re }
-    end
-    
-    # remove the "./"
-    files.map! {|fn| fn[0..1] == "./" ? fn[2..-1] : fn}
-    
-    if opt[:sort_files]
-      files = files.sort_by {|fn| fn.downcase}
-    end
-    
-    files + io_files
   end
-  
+
   def self.shebang_matches(fn)
-    File.open(fn) do |f|
-      begin
-        line = f.readline
-      rescue
-        line = ""
-      end
-      begin
-        if line =~ /^#!/
-            if line =~ /\b(ruby|perl|php|python)\b/
-              return "."+FILE_TYPES[$1.intern].first
-            elsif line =~ /\b(bash|csh|ksh|zsh)\b/
-              return ".sh"
-            end
+    begin
+      line = fn.open.readline
+      if line =~ /^#!/
+        if line =~ /\b(ruby|perl|php|python|make)[0-9.]*\b/i
+          FILE_TYPES[$1.downcase.intern].first
+        elsif line =~ /\b(sh|bash|csh|ksh|zsh)\b/
+          ".sh"
         else
-          if line =~ /\Q<?xml /
-            return ".xml"
-          end
+          ".other"
         end
-      rescue
-        return nil
+      elsif line =~ /^<\?xml\b/
+        ".xml"
+      else
+        false
       end
-      return false
+    rescue
+      nil
     end
   end
     
-  def self.search2(str, files)
+  def self.compile_pattern(str)
     if opt[:literal]
       str = Regexp.quote(str)
     end
     if opt[:match_whole_words]
-      str = "\\b" + str + "\\b"
+      str = "\\b(?:" + str + ")\\b"
     end
     if opt[:match_whole_lines]
-      str = "^" + str + "$"
+      str = "^(?:" + str + ")$"
     end
     if opt[:match_line_starts]
-      str = "^" + str
+      str = "^(?:" + str + ")"
     end
     if opt[:match_line_ends]
-      str = str + "$"
+      str = "(?:" + str + ")$"
     end
     str = str.gsub("~bang", "!").gsub("~ques", "?")
     if $use_onig
-      if opt[:ignore_case]
-        re = Oniguruma::ORegexp.new(str, :options => Oniguruma::OPTION_IGNORECASE)
-      else
-        re = Oniguruma::ORegexp.new(str)
-      end
+      flags = opt[:ignore_case] ? Oniguruma::OPTION_IGNORECASE : nil
+      Oniguruma::ORegexp.new(str, :options => flags)
     else
-      if opt[:ignore_case]
-        re = Regexp.new(str, Regexp::IGNORECASE)
-      else
-        re = Regexp.new(str)
-      end
-    end
-    files.each do |fn| 
-      match_file(re, fn)
+      flags = opt[:ignore_case] ? Regexp::IGNORECASE : nil
+      Regexp.new(str, flags)
     end
   end
   
@@ -541,67 +503,88 @@ END
   # the match method based on them. This is gives a 2x speedup over
   # the alternative.
   def self.compile_match_file
+    needs_no_more_matches = (opt[:max_count] or opt[:eval])
+    needs_inverse_count = (opt[:print_num_matches] and opt[:invert_match])
+        
     code = []
-    code << %{def self.match_file(re, fn)        }
+    code << %{def self.match_file(re, fn, file_separator)        }
     code << %{  displayed_filename = false       }
     code << %{  count = 0                        }
-    code << %{  i = 0                            }
-    code << %{  matches = []                     }
+                if needs_inverse_count
+    code << %{    inverse_count = 0             }
+                end
     code << %{  print_num = 0                    }
-
+                if needs_no_more_matches
+    code << %{    no_more_matches = false          }
+                end
+                if opt[:print_file_each_line]
+    code << %{    fn_str = (fn.is_a?(String) ? FILE_COLOUR + fn + CLEAR_COLOURS + ":" : "")      }
+                end
                 if opt[:before_context] > 0
     code << %{    before_context = []                           }
-    code << %{    before_context_size = opt[:before_context]    }
                 end
     code << %{  if fn.is_a? String              }
-    code << %{    f = File.open(fn, "r")        }
+    code << %{    f = File.open(fn, "rb")       }
     code << %{  elsif fn.is_a? IO               }
     code << %{    f = fn                        }
     code << %{  end                             }
-
-    code << %{  all = f.read                    }
-#     code << %{  p fn }
-#     code << %{  all =~ re rescue p all }
-    code << %{  if all !~ re and not opt[:invert_match] and not opt[:print_entire_line_if_no_match]               }
-                  if opt[:print_file_if_no_match]
-    code << %{      if count == 0                                }
-    code << %{        puts fn                                     }
-    code << %{      end                                 }
-                  end
-    code << %{    return                     }
-    code << %{  end                   }
   
-    code << %{  all.each_line do |line|                                          }
-    code << %{    i += 1                                                         }
-    code << %{    rest = line                                                      }
-    code << %{    while md = re.match(rest)                                   }
-    code << %{      rest = md.post_match                                          }
-                    if opt[:print_output]
-    code << %{        matches << eval(opt[:print_output])                            }
+    code << %{  f.each_line do |line|                                         }
+    code << %{    matches = []                          }
+                  
+    code << %{    unless no_more_matches }  if needs_no_more_matches
+                    if opt[:eval]
+    code << %{        $_ = line.chomp                            }
+    code << %{        $~ = nil                                            }
+    code << %{        no_more_matches = true                                          }
+    code << %{        eval_executed = false                                          }
+    code << %{        loop do                                        }
+    code << %{          if eval_executed                                                }
+    code << %{            no_more_matches = false                                           }
+    code << %{            break                                            }
+    code << %{          end                                             }
+    code << %{          eval_executed = true                                              }
+    code << %{          #{opt[:eval]}                                        }
+    code << %{          if matches.empty?                                            }
+    code << %{            $_ =~ /^.*$/ unless $~                       }
+    code << %{            matches << $~                            }
+    code << %{          end                                            }
+    code << %{          line = $_                                }
+    code << %{          no_more_matches = false                                           }
+    code << %{          break                                         }
+    code << %{        end                                           }
                     else
-    code << %{        matches << md                                               }
+                      if opt[:print_output]
+    code << %{          line.scan(re){ matches << eval(opt[:print_output]) }                        }
+                      else
+    code << %{          line.scan(re){ matches << $~  }                                           }
+                      end
                     end
                     if opt[:print_match]
-    code << %{        puts md.to_s                                               }
+    code << %{        matches.each{|md| puts md.to_s  }                                            }
                     end
-    code << %{      count += 1                                                      }
-    code << %{    end                                                             }
+    code << %{      count += matches.size                                                  }
+    code << %{    end  } if needs_no_more_matches
                   if opt[:invert_match]
                     if opt[:print_filename] 
     code << %{        unless displayed_filename                                   }
+    code << %{          print file_separator                           }
+    code << %{          file_separator.replace("\\n")                             }
     code << %{          puts FILE_COLOUR + fn + CLEAR_COLOURS                            }
     code << %{          displayed_filename = true                                   }
     code << %{        end                                                               }
                     end
     code << %{      if matches.empty?                                                 }
+                      if needs_inverse_count
+    code << %{          inverse_count += 1                                          }
+                      end
                       if opt[:print_highlighted]
                         if opt[:print_line_number]
-    code << %{            print "\#{i.to_s.rjust(4)}|"                            }
+    code << %{            print "\#{$..to_s.rjust(4)}|"                            }
                         end
-    code << %{          puts "\#{line}"                                          }
+    code << %{          puts line.expand_tabs                                    }
                       end
     code << %{      end                                                               }
-    code << %{      matches.clear                                                   }
                   else
     code << %{      if matches.empty?                                                 }
                       if opt[:print_entire_line_if_no_match]
@@ -610,9 +593,9 @@ END
                       if opt[:use_context]
                         if opt[:before_context] > 0 
     code << %{            if print_num == 0                                                 }
-    code << %{              before_context << [i, line]                                   }
-    code << %{              if before_context.length == before_context_size+1                            }
-    code << %{                before_context = before_context[1..-1]                            }
+    code << %{              before_context << [$., line]                                   }
+    code << %{              if before_context.length >  #{opt[:before_context]}                  }
+    code << %{                before_context.shift                                        }
     code << %{              end                                                               }
     code << %{            end                                                               }
                         end
@@ -620,9 +603,9 @@ END
     code << %{            print_num -= 1                                                 }
                           if opt[:print_highlighted]
                             if opt[:print_line_number]
-    code << %{                print "\#{i.to_s.rjust(4)}|"                            }
+    code << %{                print "\#{$..to_s.rjust(4)}|"                            }
                             end
-    code << %{              puts "\#{line}"                                          }
+    code << %{              puts line.expand_tabs                                }
     code << %{            end                                                        }
                         end
                       end
@@ -630,6 +613,8 @@ END
     code << %{        print_num = opt[:after_context]                                          }
                       if opt[:print_filename]
     code << %{          unless displayed_filename                                          }
+    code << %{            print file_separator                           }
+    code << %{            file_separator.replace("\\n")                             }
     code << %{            puts FILE_COLOUR + fn + CLEAR_COLOURS                            }
     code << %{            displayed_filename = true                                          }
     code << %{          end                                                               }
@@ -639,7 +624,7 @@ END
                           if opt[:print_line_number]
     code << %{              print "\#{before_i.to_s.rjust(4)}|"                            }
                           end
-    code << %{            puts before_line                                                 }
+    code << %{            puts before_line.expand_tabs                                     }
     code << %{          end                                                               }
     code << %{          before_context = []                                                 }
                       end       
@@ -648,76 +633,52 @@ END
                       end
                       if opt[:print_highlighted]
                         if opt[:print_line_number]
-    code << %{            prefix = "\#{i.to_s.rjust(4)}|"                                   }
-                        else
-                          if opt[:print_file_each_line]
-    code << %{              fn_str = (fn.is_a?(String) ? FILE_COLOUR + fn + CLEAR_COLOURS + " " : "")                            }
-    code << %{              prefix = fn_str + "\#{i.to_s.rjust(4)}|"                            }
-                          else
-    code << %{              prefix = ""                                                        }
-                          end
+    code << %{            print  "\#{$..to_s.rjust(4)}|"                                   }
+                        elsif opt[:print_file_each_line]
+    code << %{            print fn_str + "\#{$.}:"                            }
                         end
-    code << %{          print_highlighted(prefix, line, matches)                            }
+    code << %{          print_highlighted(line, matches)                            }
                       end
     code << %{      end                                                                      }
                     if opt[:max_count] 
-    code << %{        if count >= opt[:max_count]                                          }
-    code << %{          break                                                               }
-    code << %{        end                                                               }
+    code << %{        no_more_matches = true if count >= opt[:max_count]  }
+                    end
+                    if needs_no_more_matches and not needs_inverse_count
+    code << %{        break if no_more_matches and print_num == 0 }
                     end
-    code << %{      matches.clear                                                        }
                   end
     code << %{  end                                                                      }
-                if opt[:print_highlighted] and opt[:print_filename]
-    code << %{    if count > 0                                                               }
-    code << %{      puts                                                                      }
-    code << %{    end                                                                      }
-                end
     code << %{  f.close if f === File                                                        }
                 if opt[:print_num_matches]
                   if opt[:invert_match]
-    code << %{      puts "\#{fn}:\#{i-count}"                                                 }
+    code << %{      puts "\#{fn}:\#{inverse_count}"                                           }
                   else       
-    code << %{      if count > 0                                        }
-    code << %{        puts "\#{fn}:\#{count}"                           }
-    code << %{      end                                                 }
+    code << %{      puts "\#{fn}:\#{count}" if count > 0                       }
                   end
                 end
                 if opt[:print_file_if_match]
-    code << %{    if count > 0                                                               }
-    code << %{      puts fn                                                               }
-    code << %{    end                                                                      }
+    code << %{    puts fn if count > 0                                                               }
                 end
-    
                 if opt[:print_file_if_no_match]
-    code << %{    if count == 0                                                               }
-    code << %{      puts fn                                                                      }
-    code << %{    end                                                                      }
+    code << %{    puts fn if count == 0                                                               }
                 end 
     code << %{end }
-    module_eval code.join("\n")
+    module_eval code.join("\n"), "#{__FILE__} (#{__LINE__})"
   end
   
-  def self.print_highlighted(prefix, line, matches)
-    from1 = matches[0].begin(0)
-    pre = ""
-    pre = line[0..(from1-1)] if from1 > 0
-    print prefix
-    print pre
-    ptr = 0
-    0.upto(matches.length-1) do |n|
-      from = matches[n].begin(0)
-      to   = matches[n].end(0)
-      print MATCH_COLOUR +
-        line[(from+ptr)..(to+ptr-1)] +
-        CLEAR_COLOURS
-      ptr += to
-      if n == matches.length - 1
-        puts matches.last.post_match
-      else
-        print line[ptr..(matches[n+1].begin(0)-1+ptr)]
-      end
+  def self.print_highlighted(line, matches)
+    matches = matches.map{|md| md.offset(0)}
+    cuts = [0, matches, line.size].flatten
+    column = 0
+    0.upto(cuts.size-2) do |i|
+      part = line[cuts[i]...cuts[i+1]]
+      part = part.expand_tabs(column)
+      column += part.size
+      print MATCH_COLOUR if i%2 == 1
+      print part
+      print CLEAR_COLOURS if i%2 == 1
     end
+    print "\n" unless line[-1,1] == "\n"
   end
   
   def self.unknown_type(type)
@@ -743,12 +704,15 @@ Searching:
   -x, --line-regexp     Force PATTERN to match only whole lines
   -Q, --literal         Quote all metacharacters; expr is literal
   -s, --line-start      Match only at the start of a line
-  -e, --line-start      Match only at the end of a line
+  -e, --line-end        Match only at the end of a line
+
+  --eval CODE           Match with Ruby code instead of regex.
+                        [Highly experimental]
 
 Search output:
   -l, --files-with-matches
                         Only print filenames containing matches
-  -L, --files-without-match
+  -L, --files-without-matches
                         Only print filenames with no match
   -o                    Show only the part of a line matching PATTERN
                         (turns off text highlighting)
@@ -778,8 +742,9 @@ Search output:
                         Print NUM lines (default 2) of output context.
 
 File finding:
-  -f                    Only print the files found, without searching.
+  -f, --files           Only print the files found, without searching.
                         The PATTERN must not be specified.
+  --skipped             Like -f but print files that were skipped.
   --sort-files          Sort the found files lexically.
 
 File inclusion/exclusion:
@@ -810,41 +775,6 @@ format.  For example, both --type=ruby and --ruby work.
 Note that some extensions may appear in multiple types.  For example,
 .pod files are both Perl and Parrot.
 
-    --[no]asm       .s
-    --[no]cc        .c .h .xs
-    --[no]cpp       .cpp .m .h
-    --[no]csharp    .cs
-    --[no]css       .css
-    --[no]elisp     .el
-    --[no]erlang    .erl
-    --[no]fortran   .f .f77 .f90 .f95 .f03 .for .ftn .fpp
-    --[no]haskell   .hs .lhs
-    --[no]hh        .h
-    --[no]html      .htm .html .shtml
-    --[no]java      .java .properties
-    --[no]js        .js
-    --[no]jsp       .jsp .jspx .jhtm .jhtml
-    --[no]lisp      .lisp
-    --[no]make      Makefile
-    --[no]mason     .mas .mhtml .mpl .mtxt
-    --[no]ocaml     .ml .mli
-    --[no]parrot    .pir .pasm .pmc .ops .pod .pg .tg
-    --[no]perl      .pl .pm .pod .t
-    --[no]php       .php .phpt .php3 .php4 .php5
-    --[no]prolog    .pl .ecl
-    --[no]python    .py
-    --[no]ruby      .rb .rhtml .rjs .rxml Rakefile .rake .erb .haml
-    --[no]scheme    .scm
-    --[no]shell     .sh .bash .csh .ksh .zsh
-    --[no]sql       .sql .ctl
-    --[no]tcl       .tcl
-    --[no]tex       .tex .cls .sty
-    --[no]text      .txt .text
-    --[no]tt        .tt .tt2 .ttml
-    --[no]vb        .bas .cls .frm .ctl .vb .resx
-    --[no]vim       .vim
-    --[no]xml       .xml .dtd .xslt
-    --[no]yaml      .yaml .yml
 END
 
 Rak.search