ripper-tags 0.1.1 → 0.1.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTY5YzNjZGUwOTNkOGMwMzM0NTY1MjE3ZDc4MDkyNjVkNjM2ZDBhNw==
+  data.tar.gz: !binary |-
+    MWM1M2IzYTk2ODgyZjEwZjQ5ODVlZDYzN2E1Nzc1NGEzOWRmYTI0Ng==
+SHA512:
+  metadata.gz: !binary |-
+    NDM5MWNkMjk2Y2YxODE2ZTMxODcyY2I3ODcwN2M1YmY5ODFjZGU1ZWI3Mjhj
+    YmIzMzAwMDcyZWM0ZmMxZTA3MTkxYTA0ODk1N2MwY2EwMzUxMTZmZDg1ZDQz
+    YzljYWE2NzEzZDY1YTVlOGQ0MDViNjJmYzBlMzJkNmYzZThkOTU=
+  data.tar.gz: !binary |-
+    ZWU2ZDQ5Nzc0MzAyNjNjYjI0YjU5MTczMjRkMDUwZTJiZWRhN2Y2Y2YwYmQz
+    Yzg5MDg3MDJkOTMzNzIyMmI2ZjQzNzliYmIwMzIyNzliNzYzMWQxNjFlZTMz
+    MDAxNzVlZjk2YmRjMDlhYTZiYTAzMDRkYjViNjFjYTNlNDgwZGI=

data/.gitignore

@@ -0,0 +1,3 @@
+tags
+TAGS
+ripper-tags-*gem

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org'
 gemspec
 
 gem 'rake'

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Aman Gupta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -2,56 +2,26 @@
 
 fast, accurate ctags generator for ruby source code using Ripper
 
-### usage (bin)
+### usage (command-line)
+
+Typical usage:
+
+```
+ripper-tags -R --exclude=vendor
+```
+
+This parses all `*.rb` files in the current project, excluding ones in `vendor/`
+directory, and saves tags in Vim format to a file named `./tags`.
+
+To see all available options:
 
 ```
-$ ripper-tags /usr/lib/ruby/1.8/timeout.rb
-   30  module   Timeout
-   35   class   Timeout::Error < Interrupt
-   37   class   Timeout::ExitException < Exception
-   40   const   Timeout::THIS_FILE
-   41   const   Timeout::CALLER_OFFSET
-   52     def   Timeout#timeout
-  100     def   Object#timeout
-  108   const   TimeoutError
-
-$ ripper-tags --debug /usr/lib/ruby/1.8/timeout.rb
-[[:module,
-  ["Timeout", 30],
-  [[:class, ["Error", 35], ["Interrupt", 35], []],
-   [:class, ["ExitException", 37], ["Exception", 37], []],
-   [:assign, "THIS_FILE", 40],
-   [:assign, "CALLER_OFFSET", 41],
-   [:def, "timeout", 52]]],
- [:def, "timeout", 100],
- [:assign, "TimeoutError", 108]]
-
-$ ripper-tags --vim /usr/lib/ruby/1.8/timeout.rb
-!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
-!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
-CALLER_OFFSET	/usr/lib/ruby/1.8/timeout.rb	/^  CALLER_OFFSET = ((c = caller[0]) && THIS_FILE =~ c) ? 1 : 0$/;"	C	class:Timeout
-Error	/usr/lib/ruby/1.8/timeout.rb	/^  class Error < Interrupt$/;"	c	class:Timeout	inherits:Interrupt
-ExitException	/usr/lib/ruby/1.8/timeout.rb	/^  class ExitException < ::Exception # :nodoc:$/;"	c	class:Timeout	inherits:Exception
-THIS_FILE	/usr/lib/ruby/1.8/timeout.rb	/^  THIS_FILE = \/\\A#{Regexp.quote(__FILE__)}:\/o$/;"	C	class:Timeout
-Timeout	/usr/lib/ruby/1.8/timeout.rb	/^module Timeout$/;"	m
-TimeoutError	/usr/lib/ruby/1.8/timeout.rb	/^TimeoutError = Timeout::Error # :nodoc:$/;"	C	class:
-timeout	/usr/lib/ruby/1.8/timeout.rb	/^def timeout(n, e = nil, &block) # :nodoc:$/;"	f	class:Object
-timeout	/usr/lib/ruby/1.8/timeout.rb	/^  def timeout(sec, klass = nil)$/;"	f	class:Timeout
-
-$ ripper-tags --json /usr/lib/ruby/1.8/timeout.rb
-{"full_name":"Timeout","name":"Timeout","kind":"module","line":30,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"module Timeout"}
-{"full_name":"Timeout::Error","name":"Error","class":"Timeout","inherits":"Interrupt","kind":"class","line":35,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"  class Error < Interrupt"}
-{"full_name":"Timeout::ExitException","name":"ExitException","class":"Timeout","inherits":"Exception","kind":"class","line":37,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"  class ExitException < ::Exception # :nodoc:"}
-{"name":"THIS_FILE","full_name":"Timeout::THIS_FILE","class":"Timeout","kind":"constant","line":40,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"  THIS_FILE = /\\A#{Regexp.quote(__FILE__)}:/o"}
-{"name":"CALLER_OFFSET","full_name":"Timeout::CALLER_OFFSET","class":"Timeout","kind":"constant","line":41,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"  CALLER_OFFSET = ((c = caller[0]) && THIS_FILE =~ c) ? 1 : 0"}
-{"name":"timeout","full_name":"Timeout#timeout","class":"Timeout","kind":"method","line":52,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"  def timeout(sec, klass = nil)"}
-{"name":"timeout","full_name":"Object#timeout","class":"Object","kind":"method","line":100,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"def timeout(n, e = nil, &block) # :nodoc:"}
-{"name":"TimeoutError","full_name":"TimeoutError","class":"","kind":"constant","line":108,"language":"Ruby","path":"/usr/lib/ruby/1.8/timeout.rb","pattern":"TimeoutError = Timeout::Error # :nodoc:"}
+ripper-tags --help
 ```
 
 ### usage (api)
 
 ``` ruby
-require 'tag_ripper'
-tags = TagRipper.extract("def abc() end", "mycode.rb")
+require 'ripper-tags/parser'
+tags = RipperTags::Parser.extract("def abc() end", "mycode.rb")
 ```

data/bin/ripper-tags

@@ -1,98 +1,13 @@
 #!/usr/bin/env ruby
+require 'ripper-tags'
 
-require 'tag_ripper'
-require 'pp'
+trap(:INT) { abort }
+trap(:PIPE) { abort }
 
-if ARGV.delete('--json')
-  require 'yajl'
-  json = true
-end
-if ARGV.delete('--debug')
-  debug = true
-end
-if ARGV.delete('--vim')
-  vim = true
-  all_tags = []
-end
-if ARGV.delete('--emacs')
-  emacs = true
-end
-if ARGV.delete('--debug-full')
-  debug_full = true
-end
-
-ARGV.each do |file|
-  begin
-    data = File.read(file)
-    sexp = TagRipper.new(data, file).parse
-    v = TagRipper::Visitor.new(sexp, file, data)
-
-    if debug_full
-      pp Ripper.sexp(data)
-
-    elsif debug
-      pp sexp
-
-    elsif json
-      v.tags.each do |tag|
-        puts Yajl.dump(tag)
-      end
-
-    elsif emacs
-      section = []
-      v.tags.each do |tag|
-        section << "#{tag[:pattern]}\x7F#{tag[:name]}\x01#{tag[:line]},0"
-      end
-      section = section.join("\n")
-
-      print "\x0C\n#{file},#{section.bytesize}\n#{section}\n"
-
-    elsif vim
-      all_tags += v.tags
-
-    else
-      v.tags.each do |tag|
-        kind = case tag[:kind]
-        when /method$/ then 'def'
-        when /^const/  then 'const'
-        else tag[:kind]
-        end
-
-        if kind == 'class' && tag[:inherits]
-          suffix = " < #{tag[:inherits]}"
-        else
-          suffix = ''
-        end
-
-        puts "#{tag[:line].to_s.rjust(5)}  #{kind.to_s.rjust(6)}   #{tag[:full_name]}#{suffix}"
-      end
-    end
-  rescue Exception => e
-    STDERR.puts [e, file].inspect
-    raise e
-  end
-end
-
-if vim
-  puts <<-EOC
-!_TAG_FILE_FORMAT\t2\t/extended format; --format=1 will not append ;" to lines/
-!_TAG_FILE_SORTED\t1\t/0=unsorted, 1=sorted, 2=foldcase/
-EOC
-
-  all_tags.sort_by!{ |t| t[:name] }
-  all_tags.each do |tag|
-    kwargs = ''
-    kwargs << "\tclass:#{tag[:class].gsub('::','.')}" if tag[:class]
-    kwargs << "\tinherits:#{tag[:inherits].gsub('::','.')}" if tag[:inherits]
-
-    kind = case tag[:kind]
-           when 'method' then 'f'
-           when 'singleton method' then 'F'
-           when 'constant' then 'C'
-           else tag[:kind].slice(0,1)
-           end
+program_name = File.basename($0)
 
-    code = tag[:pattern].gsub('\\','\\\\\\\\').gsub('/','\\/')
-    puts "%s\t%s\t/^%s$/;\"\t%c%s" % [tag[:name], tag[:path], code, kind, kwargs]
-  end
+begin
+  RipperTags.process_args(ARGV)
+rescue Errno::ENOENT, OptionParser::InvalidOption => err
+  abort "#{program_name}: #{err.message}"
 end

data/lib/ripper-tags/data_reader.rb

@@ -0,0 +1,125 @@
+require 'pp'
+require 'ripper-tags/parser'
+
+module RipperTags
+  class FileFinder
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def exclude_patterns
+      @exclude ||= Array(options.exclude).map { |pattern|
+        if pattern.index('*')
+          Regexp.new(Regexp.escape(pattern).gsub('\\*', '[^/]+'))
+        else
+          pattern
+        end
+      }
+    end
+
+    def exclude_file?(file)
+      base = File.basename(file)
+      exclude_patterns.any? {|ex|
+        case ex
+        when Regexp then base =~ ex
+        else base == ex
+        end
+      } || exclude_patterns.any? {|ex|
+        case ex
+        when Regexp then file =~ ex
+        else file.include?(ex)
+        end
+      }
+    end
+
+    def include_file?(file)
+      (options.all_files || file =~ /\.rb\z/) && !exclude_file?(file)
+    end
+
+    def find_files(list, depth = 0)
+      list.each do |file|
+        if File.directory?(file)
+          if options.recursive
+            files = Dir.entries(file).map { |name|
+              File.join(file, name) unless '.' == name || '..' == name
+            }.compact
+            find_files(files, depth + 1) {|f| yield f }
+          end
+        else
+          yield file if include_file?(file)
+        end
+      end
+    end
+
+    def each_file
+      return to_enum(__method__) unless block_given?
+      find_files(options.files) {|f| yield f }
+    end
+  end
+
+  class DataReader
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def file_finder
+      FileFinder.new(options)
+    end
+
+    def read_file(filename)
+      str = File.open(filename, 'r:utf-8') {|f| f.read }
+      normalize_encoding(str)
+    end
+
+    def normalize_encoding(str)
+      if str.respond_to?(:encode!)
+        # strip invalid byte sequences
+        str.encode!('utf-16', :invalid => :replace, :undef => :replace)
+        str.encode!('utf-8')
+      else
+        str
+      end
+    end
+
+    def each_tag
+      return to_enum(__method__) unless block_given?
+      file_finder.each_file do |file|
+        begin
+          $stderr.puts "Parsing file #{file}" if options.verbose
+          extractor = tag_extractor(file)
+        rescue => err
+          if options.force
+            $stderr.puts "Error parsing `#{file}': #{err.message}"
+          else
+            raise err
+          end
+        else
+          extractor.tags.each do |tag|
+            yield tag
+          end
+        end
+      end
+    end
+
+    def debug_dump(obj)
+      pp(obj, $stderr)
+    end
+
+    def parse_file(contents, filename)
+      sexp = Parser.new(contents, filename).parse
+      debug_dump(sexp) if options.debug
+      sexp
+    end
+
+    def tag_extractor(file)
+      file_contents = read_file(file)
+      debug_dump(Ripper.sexp(file_contents)) if options.verbose_debug
+      sexp = parse_file(file_contents, file)
+      Visitor.new(sexp, file, file_contents)
+    end
+  end
+end

data/lib/ripper-tags/default_formatter.rb

@@ -0,0 +1,67 @@
+require 'pathname'
+
+module RipperTags
+  class DefaultFormatter
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def stdout?
+      '-' == options.tag_file_name
+    end
+
+    def with_output
+      if stdout?
+        yield $stdout
+      else
+        File.open(options.tag_file_name, 'w+') do |outfile|
+          yield outfile
+        end
+      end
+    end
+
+    def tag_file_dir
+      @tag_file_dir ||= Pathname.new(options.tag_file_name).dirname
+    end
+
+    def relative_path(tag)
+      path = tag.fetch(:path)
+      if options.tag_relative && !stdout? && path.index('/') != 0
+        Pathname.new(path).relative_path_from(tag_file_dir).to_s
+      else
+        path
+      end
+    end
+
+    def display_kind(tag)
+      case tag.fetch(:kind)
+      when /method$/ then 'def'
+      when /^const/  then 'const'
+      else tag[:kind]
+      end
+    end
+
+    def display_inheritance(tag)
+      if 'class' == tag[:kind] && tag[:inherits]
+        " < #{tag[:inherits]}"
+      else
+        ""
+      end
+    end
+
+    def format(tag)
+      "%5s  %8s   %s%s" % [
+        tag.fetch(:line).to_s,
+        display_kind(tag),
+        tag.fetch(:full_name),
+        display_inheritance(tag)
+      ]
+    end
+
+    def write(tag, io)
+      io.puts format(tag)
+    end
+  end
+end

data/lib/ripper-tags/emacs_formatter.rb

@@ -0,0 +1,69 @@
+require 'ripper-tags/default_formatter'
+require 'stringio'
+
+module RipperTags
+  # Generates etags format as described in
+  # https://en.wikipedia.org/wiki/Ctags#Etags_2
+  #
+  # The format is non-trivial since it requires section header for each source
+  # file to contain the size of tag data in bytes. This is accomplished by
+  # buffering tag definitions per-file and flushing them to target IO when a
+  # new source file is encountered or when `with_output` block finishes. This
+  # assumes that incoming tags are ordered by source file.
+  class EmacsFormatter < DefaultFormatter
+    def initialize(*)
+      super
+      @current_file = nil
+      @section_io = nil
+    end
+
+    def with_output
+      super do |io|
+        begin
+          yield io
+        ensure
+          flush_file_section(io)
+        end
+      end
+    end
+
+    def write(tag, io)
+      filename = relative_path(tag)
+      section_io = start_file_section(filename, io)
+      super(tag, section_io)
+    end
+
+    def start_file_section(filename, io)
+      if filename != @current_file
+        flush_file_section(io)
+        @current_file = filename
+        @section_io = StringIO.new
+      else
+        @section_io
+      end
+    end
+
+    def flush_file_section(out)
+      if @section_io
+        data = @section_io.string
+        out.write format_section_header(@current_file, data)
+        out.write data
+      end
+    end
+
+    def format_section_header(filename, data)
+      data_size = data.respond_to?(:bytesize) ? data.bytesize : data.size
+      "\x0C\n%s,%d\n" % [ filename, data_size ]
+    end
+
+    def format(tag)
+      pattern = tag.fetch(:pattern)
+      "%s\x7F%s\x01%d,%d" % [
+        tag.fetch(:pattern),
+        tag.fetch(:name),
+        tag.fetch(:line),
+        0,
+      ]
+    end
+  end
+end

data/lib/ripper-tags/json_formatter.rb

@@ -0,0 +1,10 @@
+require 'yajl'
+require 'ripper-tags/default_formatter'
+
+module RipperTags
+  class JSONFormatter < DefaultFormatter
+    def format(tag)
+      Yajl.dump(tag)
+    end
+  end
+end