starscope 1.0.4 → 1.1.0

data/lib/starscope/db.rb

@@ -1,20 +1,27 @@
 require 'date'
 require 'oj'
+require 'set'
 require 'zlib'
 
 require 'starscope/matcher'
 require 'starscope/output'
 require 'starscope/record'
 
-require 'starscope/langs/coffeescript'
-require 'starscope/langs/go'
-require 'starscope/langs/ruby'
-
-LANGS = [
-  StarScope::Lang::CoffeeScript,
-  StarScope::Lang::Go,
-  StarScope::Lang::Ruby
-]
+# cscope has this funky issue where it refuses to recognize function calls that
+# happen outside of a function definition - this isn't an issue in C, where all
+# calls must occur in a function, but in ruby et al. it is perfectly legal to
+# write normal code outside the "scope" of a function definition - we insert a
+# fake shim "global" function everywhere we can to work around this
+CSCOPE_GLOBAL_HACK_START = "\n\t$-\n"
+CSCOPE_GLOBAL_HACK_STOP = "\n\t}\n"
+
+# dynamically load all our language extractors
+LANGS = []
+Dir.glob("#{File.dirname(__FILE__)}/langs/*.rb").each do |path|
+  require path
+  lang = /(\w+)\.rb$/.match(path)[1]
+  LANGS << eval("StarScope::Lang::#{lang.capitalize}")
+end
 
 class StarScope::DB
 
@@ -23,8 +30,8 @@ class StarScope::DB
   class NoTableError < StandardError; end
   class UnknownDBFormatError < StandardError; end
 
-  def initialize(progress, verbose)
-    @output = StarScope::Output.new(progress, verbose)
+  def initialize(output_level)
+    @output = StarScope::Output.new(output_level)
     @meta = {:paths => [], :files => {}, :excludes => [],
              :version => StarScope::VERSION}
     @tables = {}
@@ -72,27 +79,24 @@ class StarScope::DB
   end
 
   def add_excludes(paths)
+    @output.log("Excluding files in paths #{paths}...")
     @meta[:paths] -= paths.map {|p| normalize_glob(p)}
     paths = paths.map {|p| normalize_fnmatch(p)}
     @meta[:excludes] += paths
     @meta[:excludes].uniq!
 
-    deleted = []
-    @meta[:files].delete_if do |name, record|
-      ret = matches_exclude(paths, name)
-      deleted << name if ret
-      ret
-    end
-    remove_files(deleted)
+    excluded = @meta[:files].keys.select {|name| matches_exclude?(paths, name)}
+    remove_files(excluded)
   end
 
   def add_paths(paths)
+    @output.log("Adding files in paths #{paths}...")
     @meta[:excludes] -= paths.map {|p| normalize_fnmatch(p)}
     paths = paths.map {|p| normalize_glob(p)}
     @meta[:paths] += paths
     @meta[:paths].uniq!
     files = Dir.glob(paths).select {|f| File.file? f}
-    files.delete_if {|f| matches_exclude(@meta[:excludes], f)}
+    files.delete_if {|f| matches_exclude?(@meta[:excludes], f)}
     return if files.empty?
     @output.new_pbar("Building", files.length)
     add_new_files(files)
@@ -100,40 +104,33 @@ class StarScope::DB
   end
 
   def update
-    new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].keys
-    new_files.delete_if {|f| matches_exclude(@meta[:excludes], f)}
+    changes = @meta[:files].keys.group_by {|name| file_changed(name)}
+    changes[:modified] ||= []
+    changes[:deleted] ||= []
 
-    @output.new_pbar("Updating", new_files.length + @meta[:files].length)
-    changed = false
-    @tables, tmp_tbls = {}, @tables
-    to_prune = []
-
-    @meta[:files].delete_if do |name, record|
-      @output.log("Updating `#{name}`")
-
-      event = file_event(name, record)
-      if event != :unchanged
-        to_prune << name
-        changed = true
-        parse_file(name) if event == :modified
-      end
+    new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].keys
+    new_files.delete_if {|f| matches_exclude?(@meta[:excludes], f)}
 
-      @output.inc_pbar
-      event == :deleted
+    if changes[:deleted].empty? && changes[:modified].empty? && new_files.empty?
+      @output.print("No changes detected.")
+      return false
     end
 
-    @tables, tmp_tbls = tmp_tbls, @tables
-    remove_files(to_prune)
-    merge_db(tmp_tbls)
-
+    @output.new_pbar("Updating", changes[:modified].length + new_files.length)
+    remove_files(changes[:deleted])
+    update_files(changes[:modified])
     add_new_files(new_files)
     @output.finish_pbar
-    changed || !new_files.empty?
+
+    true
   end
 
   def dump_table(table)
     raise NoTableError if not @tables[table]
+
     puts "== Table: #{table} =="
+    puts "No records" if @tables[table].empty?
+
     @tables[table].sort {|a,b|
       a[:name][-1].to_s.downcase <=> b[:name][-1].to_s.downcase
     }.each do |record|
@@ -208,36 +205,42 @@ END
       next if lines.empty?
 
       buf << "\t@#{filename}\n\n"
+      buf << "0 #{CSCOPE_GLOBAL_HACK_START}\n"
       files << filename
+      func_count = 0
 
       lines.sort.each do |line_no, records|
-        begin
-          line = records.first[:line].strip.gsub(/\s+/, ' ')
-        rescue ArgumentError
-          # invalid utf-8 byte sequence in the line, just do our best
-          line = records.first[:line]
-        end
-
-        toks = {}
-        records.each do |record|
-          key = record[:name][-1].to_s
-          index = line.index(key)
-          while index
-            toks[index] = record
-            index = line.index(key, index + 1)
-          end
-        end
+        line = records.first[:line]
+        toks = tokenize_line(line, records)
         next if toks.empty?
 
         prev = 0
         buf << line_no.to_s << " "
-        toks.sort.each do |offset, record|
-          key = record[:name][-1].to_s
-          buf << line.slice(prev...offset) << "\n"
-          buf << StarScope::Record.cscope_mark(record[:tbl], record) << key << "\n"
-          prev = offset + key.length
+        toks.each do |offset, record|
+
+          # Don't export nested functions, cscope barfs on them since C doesn't
+          # have them at all. Skipping tokens is easy; since prev isn't updated
+          # they get turned into plain text automatically.
+          if record[:type] == :func
+            case record[:tbl]
+            when :defs
+              func_count += 1
+              next unless func_count == 1
+            when :end
+              func_count -= 1
+              next unless func_count == 0
+            end
+          end
+
+          buf << CSCOPE_GLOBAL_HACK_STOP if record[:type] == :func && record[:tbl] == :defs
+          buf << cscope_plaintext(line, prev, offset) << "\n"
+          buf << StarScope::Record.cscope_mark(record[:tbl], record) << record[:key] << "\n"
+          buf << CSCOPE_GLOBAL_HACK_START if record[:type] == :func && record[:tbl] == :end
+
+          prev = offset + record[:key].length
+
         end
-        buf << line.slice(prev..-1) << "\n\n"
+        buf << cscope_plaintext(line, prev, line.length) << "\n\n"
       end
     end
 
@@ -264,12 +267,27 @@ END
   def add_new_files(files)
     files.each do |file|
       @output.log("Adding `#{file}`")
-      @meta[:files][file] = {}
       parse_file(file)
       @output.inc_pbar
     end
   end
 
+  def update_files(files)
+    remove_files(files)
+    add_new_files(files)
+  end
+
+  def remove_files(files)
+    files.each do |file|
+      @output.log("Removing `#{file}`")
+      @meta[:files].delete(file)
+    end
+    files = files.to_set
+    @tables.each do |name, tbl|
+      tbl.delete_if {|val| files.include?(val[:file])}
+    end
+  end
+
   # File.fnmatch treats a "**" to match files and directories recursively
   def normalize_fnmatch(path)
     if path == "."
@@ -307,12 +325,59 @@ END
     return db
   end
 
-  def matches_exclude(patterns, file)
+  def tokenize_line(line, records)
+    toks = {}
+
+    records.each do |record|
+      key = record[:name][-1].to_s
+
+      # use the column if we have it, otherwise fall back to scanning
+      index = record[:col] || line.index(key)
+
+      # keep scanning if our current index doesn't actually match the key, or if
+      # either the preceeding or succeeding character is a word character
+      # (meaning we've accidentally matched the middle of some other token)
+      while !index.nil? &&
+        ((line[index, key.length] != key) ||
+         (index > 0 && line[index-1] =~ /\w/) ||
+         (index+key.length < line.length && line[index+key.length] =~ /\w/))
+        index = line.index(key, index+1)
+      end
+
+      next if index.nil?
+
+      # Strip trailing non-word characters, otherwise cscope barfs on
+      # function names like `include?`
+      if key =~ /^\W*$/
+        next unless [:defs, :end].include?(record[:tbl])
+      else
+        key.sub!(/\W+$/, '')
+      end
+
+      record[:key] = key
+      toks[index] = record
+
+    end
+
+    return toks.sort
+  end
+
+  def cscope_plaintext(line, start, stop)
+    ret = line.slice(start, stop-start)
+    ret.lstrip! if start == 0
+    ret.rstrip! if stop == line.length
+    ret.gsub(/\s+/, ' ')
+  rescue ArgumentError
+    # invalid utf-8 byte sequence in the line, oh well
+    line
+  end
+
+  def matches_exclude?(patterns, file)
     patterns.map {|p| File.fnmatch(p, file)}.any?
   end
 
   def parse_file(file)
-    @meta[:files][file][:last_updated] = File.mtime(file).to_i
+    @meta[:files][file] = {:last_updated => File.mtime(file).to_i}
 
     LANGS.each do |lang|
       next if not lang.match_file file
@@ -321,27 +386,14 @@ END
         @tables[tbl] << StarScope::Record.build(file, name, args)
       end
       @meta[:files][file][:lang] = lang.name.split('::').last.to_sym
+      return
     end
   end
 
-  def merge_db(new_tbls)
-    new_tbls.each do |name, tbl|
-      @tables[name] ||= []
-      @tables[name].concat(tbl)
-    end
-  end
-
-  def remove_files(files)
-    check = Hash[files.map {|f| [f, true]}]
-    @tables.each do |name, tbl|
-      tbl.delete_if {|val| check[val[:file]]}
-    end
-  end
-
-  def file_event(name, record)
+  def file_changed(name)
     if not File.exists?(name) or not File.file?(name)
       :deleted
-    elsif record[:last_updated] < File.mtime(name).to_i
+    elsif @meta[:files][name][:last_updated] < File.mtime(name).to_i
       :modified
     else
       :unchanged

data/lib/starscope/langs/coffeescript.rb

@@ -1,5 +1,5 @@
 module StarScope::Lang
-  module CoffeeScript
+  module Coffeescript
 
     def self.match_file(name)
       name.end_with?(".coffee")

data/lib/starscope/langs/go.rb

@@ -60,6 +60,7 @@ module StarScope::Lang
             stack.pop
           when /(.+)\s*=.*/
             parse_def($1, line_no, scope, &block)
+            parse_call(line, line_no, scope, &block)
           else
             parse_def(line, line_no, scope, &block)
           end
@@ -122,10 +123,12 @@ module StarScope::Lang
         stack.push(:def)
       when /^var\s+(\w+)\s/
         yield :defs, scope + [$1], :line_no => line_no
+        parse_call(line, line_no, scope, &block)
       when /^const\s+\(/
         stack.push(:def)
       when /^const\s+(\w+)\s/
         yield :defs, scope + [$1], :line_no => line_no
+        parse_call(line, line_no, scope, &block)
       when /^\s+(.*?) :?=[^=]/
         $1.split(' ').each do |var|
           next if CONTROL_KEYS.include?(var)

data/lib/starscope/langs/ruby.rb

@@ -55,34 +55,34 @@ module StarScope::Lang
 
         case node.type
         when :send
-          yield :calls, scoped_name(node), :line_no => loc.expression.line
+          yield :calls, scoped_name(node), :line_no => loc.line, :col => loc.column
           if node.children[0].nil? and node.children[1] == :require and node.children[2].type == :str
             yield :requires, node.children[2].children[0].split("/"),
-              :line_no => loc.expression.line
+              :line_no => loc.line, :col => loc.column
           end
 
         when :def
           yield :defs, @scope + [node.children[0]],
-            :line_no => loc.expression.line, :type => :func
-          yield :end, :end, :line_no => loc.end.line, :type => :func
+            :line_no => loc.line, :type => :func, :col => loc.name.column
+          yield :end, :end, :line_no => loc.end.line, :type => :func, :col => loc.end.column
 
         when :defs
           yield :defs, @scope + [node.children[1]],
-            :line_no => loc.expression.line, :type => :func
-          yield :end, :end, :line_no => loc.end.line, :type => :func
+            :line_no => loc.line, :type => :func, :col => loc.name.column
+          yield :end, :end, :line_no => loc.end.line, :type => :func, :col => loc.end.column
 
         when :module, :class
           yield :defs, @scope + scoped_name(node.children[0]),
-            :line_no => loc.expression.line, :type => node.type
-          yield :end, :end, :line_no => loc.end.line, :type => node.type
+            :line_no => loc.line, :type => node.type, :col => loc.name.column
+          yield :end, :end, :line_no => loc.end.line, :type => node.type, :col => loc.end.column
 
         when :casgn
           fqn = scoped_name(node)
-          yield :assigns, fqn, :line_no => loc.expression.line
-          yield :defs, fqn, :line_no => loc.expression.line
+          yield :assigns, fqn, :line_no => loc.line, :col => loc.name.column
+          yield :defs, fqn, :line_no => loc.line, :col => loc.name.column
 
         when :lvasgn, :ivasgn, :cvasgn, :gvasgn
-          yield :assigns, @scope + [node.children[0]], :line_no => loc.expression.line
+          yield :assigns, @scope + [node.children[0]], :line_no => loc.line, :col => loc.name.column
         end
       end
 

data/lib/starscope/output.rb

@@ -4,14 +4,13 @@ class StarScope::Output
 
   PBAR_FORMAT = '%t: %c/%C %E ||%b>%i||'
 
-  def initialize(progress, verbose)
-    @progress = progress
-    @verbose = verbose
+  def initialize(level)
+    @level = level
     @pbar = nil
   end
 
   def new_pbar(title, num_items)
-    if @progress
+    if @level != :quiet
       @pbar = ProgressBar.create(:title => title, :total => num_items,
                                 :format => PBAR_FORMAT, :length => 80)
     end
@@ -22,15 +21,23 @@ class StarScope::Output
   end
 
   def finish_pbar
-    if @pbar
-      @pbar.finish
-      @pbar = nil
-    end
+    @pbar.finish if @pbar
+    @pbar = nil
   end
 
   def log(msg)
-    return if not @verbose
+    return if @level != :verbose
+    output(msg)
+  end
+
+  def print(msg)
+    return if @level == :quiet
+    output(msg)
+  end
+
+  private
 
+  def output(msg)
     if @pbar
       @pbar.log(msg)
     else

data/lib/starscope/version.rb

@@ -1,3 +1,3 @@
 module StarScope
-  VERSION = "1.0.4"
+  VERSION = "1.1.0"
 end