starscope 0.0.8 → 0.1.0

data/CHANGELOG.md

@@ -1,7 +1,23 @@
 Changelog
 =========
 
-v0.0.8 (trunk)
+v0.1.0 (trunk)
+-------------------
+
+New Features:
+ * Progress bar when building or updating database.
+
+Bug Fixes:
+ * Handle the case when a ruby file produces a nil parse tree.
+ * Many misc fixes.
+
+Internals:
+ * Another new version of the ruby parser.
+ * Replace the default JSON module with Oj, which is more than twice as fast for
+   large databases.
+ * Misc optimizations.
+
+v0.0.8 (2013-07-20)
 -------------------
 
 Bug Fixes:

data/Gemfile.lock

@@ -1,8 +1,10 @@
 PATH
   remote: .
   specs:
-    starscope (0.0.8)
-      parser (= 2.0.0.pre2)
+    starscope (0.1.0)
+      oj (~> 2.1)
+      parser (= 2.0.0.pre6)
+      ruby-progressbar (~> 1.1)
 
 GEM
   remote: https://rubygems.org/
@@ -17,7 +19,8 @@ GEM
     debugger-linecache (1.2.0)
     debugger-ruby_core_source (1.2.2)
     method_source (0.8.1)
-    parser (2.0.0.pre2)
+    oj (2.1.4)
+    parser (2.0.0.pre6)
       ast (~> 1.1)
       slop (~> 3.4, >= 3.4.5)
     pry (0.9.12.2)
@@ -28,6 +31,7 @@ GEM
       debugger (~> 1.3)
       pry (~> 0.9.10)
     rake (10.0.4)
+    ruby-progressbar (1.1.1)
     slop (3.4.5)
 
 PLATFORMS

data/README.md

@@ -1,6 +1,8 @@
 StarScope
 =========
 
+https://rubygems.org/gems/starscope
+
 *Note to those users looking at the GitHub language statistics: there is no perl
 here, that is their bug to fix.*
 

data/TODO.md

@@ -1,6 +1,7 @@
 TODO
 =========
 
+* Add --quiet to hide progress bar and --verbose to print each file
 * Parallelization, perhaps via https://github.com/grosser/parallel
 * Export advanced ctags annotations
 * Export to cscope format

data/bin/starscope

@@ -72,9 +72,10 @@ end
 def run_query(db, table, value)
   if not value
     $stderr.puts "Invalid input - no query found."
-    return
+    return false
   end
-  puts db.query(table.to_sym, value).map {|val| StarScope::Datum.to_s(val)}
+  key, results = db.query(table.to_sym, value)
+  puts results.map {|val| StarScope::Datum.to_s(key, val)} if results
   return true
 rescue StarScope::DB::NoTableError
   $stderr.puts "Table '#{table}' doesn't exist."
@@ -95,6 +96,7 @@ end
 
 if options[:auto] and not options[:write]
   options[:write] = DEFAULT_DB
+  options[:auto_write] = true
 end
 
 if File.exists?(DEFAULT_DB) and not options[:read]
@@ -105,16 +107,21 @@ db = StarScope::DB.new
 
 if options[:read]
   db.load(options[:read])
-  db.add_dirs(ARGV)
+  if !ARGV.empty?
+    db.add_paths(ARGV)
+    new_data = true
+  end
 elsif ARGV.empty?
-  db.add_dirs(['.'])
+  db.add_paths(['.'])
+  new_data = true
 else
-  db.add_dirs(ARGV)
+  db.add_paths(ARGV)
+  new_data = true
 end
 
-db.update if options[:read] and options[:auto]
+changed = db.update if options[:read] and options[:auto]
 
-db.save(options[:write]) if options[:write]
+db.save(options[:write]) if options[:write] and (new_data or changed or not options[:auto_write])
 
 if options[:export]
   format, path = options[:export].split(',', 2)
@@ -166,8 +173,8 @@ if options[:linemode]
       when "summary"
         print_summary(db)
       when "update"
-        db.update
-        db.save(options[:write]) if options[:write]
+        changed = db.update
+        db.save(options[:write]) if options[:write] and changed
       when "help"
         puts linemode_help
       when "version"

data/lib/starscope/datum.rb

@@ -1,7 +1,6 @@
 class StarScope::Datum
 
-  def self.build(key, file, args)
-    args[:key]  = key
+  def self.build(file, args)
     args[:file] = file
 
     if args[:line_no]
@@ -35,14 +34,14 @@ class StarScope::Datum
     "#{dat[:file]}:#{dat[:line_no]}"
   end
 
-  def self.to_s(dat)
+  def self.to_s(key, dat)
     str = ""
     str << "#{dat[:scope].join " "} " unless dat[:scope].empty?
-    str << "#{dat[:key]} -- #{location dat}"
+    str << "#{key} -- #{location dat}"
     str << " (#{dat[:line].strip})"
   end
 
-  def self.ctag_line(dat)
-    "#{dat[:key]}\t#{dat[:file]}\t/^#{dat[:line]}$/;"
+  def self.ctag_line(key, dat)
+    "#{key}\t#{dat[:file]}\t/^#{dat[:line]}$/;"
   end
 end

data/lib/starscope/db.rb

@@ -1,20 +1,22 @@
 require 'starscope/langs/ruby'
 require 'starscope/datum'
 require 'date'
-require 'json'
+require 'oj'
 require 'zlib'
+require 'ruby-progressbar'
 
 LANGS = [StarScope::Lang::Ruby]
 
 class StarScope::DB
 
   DB_FORMAT = 3
+  PBAR_FORMAT = '%t: %c/%C %E ||%b>%i||'
 
   class NoTableError < StandardError; end
   class UnknownDBFormatError < StandardError; end
 
   def initialize
-    @dirs = []
+    @paths = []
     @files = {}
     @tables = {}
   end
@@ -24,16 +26,16 @@ class StarScope::DB
       Zlib::GzipReader.wrap(file) do |file|
         format = file.gets.to_i
         if format == DB_FORMAT
-          @dirs   = JSON.parse(file.gets, symbolize_names: false)
-          @files  = JSON.parse(file.gets, symbolize_names: false)
-          @tables = JSON.parse(file.gets, symbolize_names: true)
+          @paths  = Oj.load(file.gets)
+          @files  = Oj.load(file.gets)
+          @tables = Oj.load(file.gets, symbol_keys: true)
         elsif format <= 2
           # Old format (pre-json), so read the directories segment then rebuild
           len = file.gets.to_i
-          add_dirs(Marshal::load(file.read(len)))
+          add_paths(Marshal::load(file.read(len)))
         elsif format < DB_FORMAT
           # Old format, so read the directories segment then rebuild
-          add_dirs(JSON.parse(file.gets, symbolize_names: false))
+          add_paths(Oj.load(file.gets))
         elsif format > DB_FORMAT
           raise UnknownDBFormatError
         end
@@ -45,26 +47,39 @@ class StarScope::DB
     File.open(file, 'w') do |file|
       Zlib::GzipWriter.wrap(file) do |file|
         file.puts DB_FORMAT
-        file.puts JSON.fast_generate @dirs
-        file.puts JSON.fast_generate @files
-        file.puts JSON.fast_generate @tables
+        file.puts Oj.dump @paths
+        file.puts Oj.dump @files
+        file.puts Oj.dump @tables
       end
     end
   end
 
-  def add_dirs(dirs)
-    @dirs += dirs
-    dirs.each do |dir|
-      Dir["#{dir}/**/*"].each do |file|
-        add_file(file)
-      end
+  def add_paths(paths)
+    paths -= @paths
+    return if paths.empty?
+    @paths += paths
+    files = paths.map {|p| self.class.files_from_path(p)}.flatten
+    return if files.empty?
+    pbar = ProgressBar.create(title: "Building", total: files.length, format: PBAR_FORMAT, length: 80)
+    files.each do |f|
+      add_file(f)
+      pbar.increment
     end
   end
 
   def update
-    @files.keys.each {|f| update_file(f)}
-    cur_files = @dirs.each {|d| Dir["#{d}/**/*"]}.flatten
-    (cur_files - @files.keys).each {|f| add_file(f)}
+    new_files = (@paths.map {|p| self.class.files_from_path(p)}.flatten) - @files.keys
+    pbar = ProgressBar.create(title: "Updating", total: new_files.length + @files.length, format: PBAR_FORMAT, length: 80)
+    changed = @files.keys.map do |f|
+      changed = update_file(f)
+      pbar.increment
+      changed
+    end
+    new_files.each do |f|
+      add_file(f)
+      pbar.increment
+    end
+    changed.any? || !new_files.empty?
   end
 
   def dump_table(table)
@@ -74,7 +89,7 @@ class StarScope::DB
       puts "#{val}"
       data.each do |datum|
         print "\t"
-        puts StarScope::Datum.to_s(datum)
+        puts StarScope::Datum.to_s(val, datum)
       end
     end
   end
@@ -96,15 +111,17 @@ class StarScope::DB
   def query(table, value)
     fqn = value.split("::")
     raise NoTableError if not @tables[table]
-    results = @tables[table][fqn.last.to_sym]
+    key = fqn.last.to_sym
+    results = @tables[table][key]
     return [] if results.nil? || results.empty?
     results.sort! do |a,b|
       StarScope::Datum.score_match(b, fqn) <=> StarScope::Datum.score_match(a, fqn)
     end
     best_score = StarScope::Datum.score_match(results[0], fqn)
-    results.select do |result|
+    results = results.select do |result|
       best_score - StarScope::Datum.score_match(result, fqn) < 4
     end
+    return key, results
   end
 
   def export_ctags(filename)
@@ -120,7 +137,7 @@ END
       defs = (@tables[:defs] || []).sort {|a,b| a <=> b}
       defs.each do |key, val|
         val.each do |entry|
-          file.puts StarScope::Datum.ctag_line(entry)
+          file.puts StarScope::Datum.ctag_line(key, entry)
         end
       end
     end
@@ -128,6 +145,16 @@ END
 
   private
 
+  def self.files_from_path(path)
+    if File.file?(path)
+      [path]
+    elsif File.directory?(path)
+      Dir[File.join(path, "**", "*")].select {|p| File.file?(p)}
+    else
+      []
+    end
+  end
+
   def add_file(file)
     return if not File.file? file
 
@@ -138,7 +165,7 @@ END
       lang.extract file do |tbl, key, args|
         @tables[tbl] ||= {}
         @tables[tbl][key] ||= []
-        @tables[tbl][key] << StarScope::Datum.build(key, file, args)
+        @tables[tbl][key] << StarScope::Datum.build(file, args)
       end
     end
   end
@@ -153,11 +180,15 @@ END
   end
 
   def update_file(file)
-    if not File.exists?(file)
+    if not File.exists?(file) or not File.file?(file)
       remove_file(file)
+      true
     elsif DateTime.parse(@files[file]).to_time < File.mtime(file)
       remove_file(file)
       add_file(file)
+      true
+    else
+      false
     end
   end
 

data/lib/starscope/langs/ruby.rb

@@ -3,7 +3,7 @@ require "parser/current"
 module StarScope::Lang
   module Ruby
     def self.match_file(name)
-      name =~ /.*\.rb/
+      name =~ /.*\.rb$/
     end
 
     def self.extract(file, &block)
@@ -24,7 +24,7 @@ module StarScope::Lang
       end
 
       def extract(&block)
-        extract_tree(@ast, &block)
+        extract_tree(@ast, &block) if not @ast.nil?
       end
 
       private
@@ -44,37 +44,36 @@ module StarScope::Lang
       end
 
       def extract_node(node)
-        if node.type == :send
+        case node.type
+        when :send
           fqn = scoped_name(node)
-          yield :calls, fqn.last, line_no: node.location.expression.line,
+          yield :calls, fqn.last,
+            line_no: node.location.expression.line,
             scope: fqn[0...-1]
-
           if node.children[0].nil? and node.children[1] == :require and node.children[2].type == :str
             fqn = node.children[2].children[0].split("/")
-            yield :requires, fqn.last, line_no: node.location.expression.line,
+            yield :requires, fqn.last,
+              line_no: node.location.expression.line,
               scope: fqn[0...-1]
           end
-        end
-
-        if node.type == :def
+        when :def
           yield :defs, node.children[0],
             line_no: node.location.expression.line,
             scope: @scope
-        elsif node.type == :defs
+        when :defs
           yield :defs, node.children[1],
             line_no: node.location.expression.line,
             scope: @scope
-        elsif [:module, :class].include? node.type
+        when :module, :class
           fqn = @scope + scoped_name(node.children[0])
           yield :defs, fqn.last, line_no: node.location.expression.line,
             scope: fqn[0...-1]
-        end
-
-        if node.type == :casgn
+        when :casgn
           fqn = scoped_name(node)
-          yield :assigns, fqn.last, line_no: node.location.expression.line,
+          yield :assigns, fqn.last,
+            line_no: node.location.expression.line,
             scope: fqn[0...-1]
-        elsif [:lvasgn, :ivasgn, :cvasgn, :gvasgn].include? node.type
+        when :lvasgn, :ivasgn, :cvasgn, :gvasgn
           yield :assigns, node.children[0],
             line_no: node.location.expression.line,
             scope: @scope

data/lib/starscope/version.rb

@@ -1,3 +1,3 @@
 module StarScope
-  VERSION = "0.0.8"
+  VERSION = "0.1.0"
 end

data/starscope.gemspec

@@ -13,7 +13,9 @@ Gem::Specification.new do |gem|
   gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ["lib"]
 
-  gem.add_dependency 'parser', '= 2.0.0.pre2'
+  gem.add_dependency 'oj', '~> 2.1'
+  gem.add_dependency 'parser', '= 2.0.0.pre6'
+  gem.add_dependency 'ruby-progressbar', '~> 1.1'
   gem.add_development_dependency 'bundler', '~> 1.3'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'pry'

metadata

@@ -1,96 +1,128 @@
 --- !ruby/object:Gem::Specification
 name: starscope
 version: !ruby/object:Gem::Version
-  version: 0.0.8
   prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Evan Huus
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-20 00:00:00.000000000 Z
+date: 2013-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: parser
-  requirement: !ruby/object:Gem::Requirement
+  prerelease: false
+  name: oj
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre2
-  type: :runtime
+        version: '2.1'
+    none: false
+- !ruby/object:Gem::Dependency
   prerelease: false
+  name: parser
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre6
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre2
+        version: 2.0.0.pre6
+    none: false
 - !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
+  prerelease: false
+  name: ruby-progressbar
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :development
+        version: '1.1'
+    none: false
+- !ruby/object:Gem::Dependency
   prerelease: false
+  name: bundler
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: rake
-  requirement: !ruby/object:Gem::Requirement
-    none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: pry
-  requirement: !ruby/object:Gem::Requirement
-    none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: pry-debugger
-  requirement: !ruby/object:Gem::Requirement
-    none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 description: A tool like the venerable cscope, but for ruby and other languages
 email: eapache@gmail.com
 executables:
@@ -122,23 +154,23 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
       segments:
       - 0
-      hash: -2279719336139896086
-required_rubygems_version: !ruby/object:Gem::Requirement
+      hash: -4533453239802094474
+      version: '0'
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
       segments:
       - 0
-      hash: -2279719336139896086
+      hash: -4533453239802094474
+      version: '0'
+  none: false
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23