starscope 0.1.6 → 0.1.7

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+
+rvm:
+    - 1.9.3
+    - 2.0.0
+    - 2.1.0

data/CHANGELOG.md

@@ -1,11 +1,22 @@
 Changelog
 =========
 
-v0.1.6 (trunk)
+v0.1.7 (trunk)
 -------------------
 
-Bug Fixes:
- * Much improved recognition of golang function calls.
+Improvements:
+ * Much better recognition and parsing of definitions and assignments to
+   variables and constants in golang code (fixes #11 among others).
+
+Infrastructure:
+ * Test suite and continuous integration (via Minitest and Travis-CI).
+ * Use github's tickets instead of a TODO file.
+
+v0.1.6 (2014-01-10)
+-------------------
+
+Improvements:
+ * Better recognition of golang function calls.
 
 v0.1.5 (2014-01-06)
 -------------------

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    starscope (0.1.6)
+    starscope (0.1.7)
       oj (~> 2.5)
       parser (~> 2.1)
       ruby-progressbar (~> 1.4)
@@ -20,7 +20,7 @@ GEM
     debugger-ruby_core_source (1.3.1)
     method_source (0.8.2)
     oj (2.5.3)
-    parser (2.1.3)
+    parser (2.1.4)
       ast (~> 1.1)
       slop (~> 3.4, >= 3.4.5)
     pry (0.9.12.4)

data/README.md

@@ -2,6 +2,7 @@ StarScope
 =========
 
 [![Gem Version](https://badge.fury.io/rb/starscope.png)](http://badge.fury.io/rb/starscope)
+[![Build Status](https://travis-ci.org/eapache/starscope.png?branch=master)](https://travis-ci.org/eapache/starscope)
 
 Anyone who has done much programming in C (or C++) on a unix-based OS has come
 across the fantastic [Cscope](http://cscope.sourceforge.net/) tool. Sadly, it

data/Rakefile

@@ -1 +1,10 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib/starscope'
+  t.test_files = FileList['test/lib/test_*.rb']
+end
+
+desc "Run tests"
+task :default => :test

data/lib/starscope/langs/go.rb

@@ -1,10 +1,13 @@
 module StarScope::Lang
   module Go
     FUNC_CALL = /([\w\.]*?\w)\(/
+    END_OF_BLOCK = /^\s*}\s*$/
+    END_OF_GROUP = /^\s*\)\s*$/
     BUILTIN_FUNCS = ['new', 'make', 'len', 'close', 'copy', 'delete',
                      'int', 'int8', 'int16', 'int32', 'int64',
                      'uint', 'uint8', 'uint16', 'uint32', 'uint64',
                      'string', 'byte']
+    CONTROL_KEYS = ['if', 'for', 'switch', 'case']
 
     def self.match_file(name)
       name =~ /.*\.go$/
@@ -47,36 +50,38 @@ module StarScope::Lang
         case stack[-1]
         when :struct
           case line
-          when /\s*(\w+)\s+\w+/
-            yield :defs, $1, line_no: line_no, scope: scope
-          when /}/
+          when END_OF_BLOCK
             yield :end, "}", line_no: line_no, scope: scope, type: :class
             stack.pop
             scope.pop
+          when /(.+)\s+\w+/
+            parse_def($1, line_no, scope, &block)
           end
         when :interface
           case line
-          when /\s*(\w+)\(.*\)\s+/
-            yield :defs, $1, line_no: line_no, scope: scope
-          when /}/
+          when END_OF_BLOCK
             yield :end, "}", line_no: line_no, scope: scope, type: :class
             stack.pop
             scope.pop
+          when /(\w+)\(.*\)\s+/
+            yield :defs, $1, line_no: line_no, scope: scope
           end
         when :def
           case line
-          when /\s*(\w+)\s+/
-            yield :defs, $1, line_no: line_no, scope: scope
-          when /\)/
+          when END_OF_GROUP
             stack.pop
+          when /(.+)\s*=.*/
+            parse_def($1, line_no, scope, &block)
+          else
+            parse_def(line, line_no, scope, &block)
           end
         when :import
           case line
+          when END_OF_GROUP
+            stack.pop
           when /"(.+)"/
             name = $1.split('/')
             yield :imports, name[-1], line_no: line_no, scope: name[0...-1]
-          when /\)/
-            stack.pop
           end
         else
           if stack[-1] == :func and /^}/ =~ line
@@ -116,15 +121,15 @@ module StarScope::Lang
             stack.push(:def)
           when /^const\s+(\w+)\s+\w+/
             yield :defs, $1, line_no: line_no, scope: scope
-          when /^\s+(.*?) :?=.*/
-            $1.split(',').each do |var|
-              var = var.strip
-              name = var.split('.')
-              case name.length
-              when 1
+          when /^\s+(.*?) :?=[^=]/
+            $1.split(' ').each do |var|
+              next if CONTROL_KEYS.include?(var)
+              name = var.delete(',').split('.')
+              next if name[0] == "_" # assigning to _ is a discard in golang
+              if name.length == 1
                 yield :assigns, name[0], line_no: line_no, scope: scope
-              when 2
-                yield :assigns, name[1], line_no: line_no, scope: [name[0]]
+              else
+                yield :assigns, name[1], line_no: line_no, scope: name[0..-1]
               end
             end
             parse_call(line, line_no, scope, &block)
@@ -155,5 +160,12 @@ module StarScope::Lang
         end
       end
     end
+
+    def self.parse_def(line, line_no, scope)
+      line.split.each do |var|
+        yield :defs, var.delete(','), line_no: line_no, scope: scope
+        break if not var.end_with?(',')
+      end
+    end
   end
 end

data/lib/starscope/version.rb

@@ -1,3 +1,3 @@
 module StarScope
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/starscope.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
   gem.license       = 'MIT'
   gem.files         = `git ls-files`.split("\n")
   gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.test_files    = `git ls-files -- test/*`.split("\n")
   gem.require_paths = ["lib"]
   gem.required_ruby_version = '>= 1.9.3'
 

data/test/{test.go → files/sample_golang.go}

@@ -1,5 +1,14 @@
 package main
 
+var (
+	v1, v2 int = 3, 4
+)
+
+const (
+	Sunday = iota
+	Monday
+)
+
 func a(c int) int {
 	return 3
 }
@@ -12,6 +21,10 @@ func c(a, b int) int {
 	return 1
 }
 
+func ttt() (int, int) {
+	return 1, 2
+}
+
 func main() {
 	var (
 		q int
@@ -20,6 +33,12 @@ func main() {
 	x := a(1)
 	y := b()
 	z := c(a(q), b())
+	n, m := ttt()
+	m, x = ttt()
+
+	if m == x {
+		v1 = v2
+	}
 
 	a(c(b(), b()))
 }

data/test/files/sample_ruby.rb

@@ -0,0 +1,163 @@
+require 'date'
+require 'zlib'
+
+LANGS = [
+  StarScope::Lang::Go,
+  StarScope::Lang::Ruby
+]
+
+class StarScope::DB
+
+  PBAR_FORMAT = '%t: %c/%C %E ||%b>%i||'
+
+  class NoTableError < StandardError; end
+
+  def initialize(progress)
+    @progress = progress
+    @paths = []
+    @files = {}
+    @tables = {}
+  end
+
+  def load(file)
+    File.open(file, 'r') do |file|
+      Zlib::GzipReader.wrap(file) do |file|
+        format = file.gets.to_i
+        if format == DB_FORMAT
+          @paths  = Oj.load(file.gets)
+          @files  = Oj.load(file.gets)
+          @tables = Oj.load(file.gets, symbol_keys: true)
+        elsif format <= 2
+          # Old format (pre-json), so read the directories segment then rebuild
+          len = file.gets.to_i
+          add_paths(Marshal::load(file.read(len)))
+        elsif format < DB_FORMAT
+          # Old format, so read the directories segment then rebuild
+          add_paths(Oj.load(file.gets))
+        elsif format > DB_FORMAT
+          raise UnknownDBFormatError
+        end
+      end
+    end
+  end
+
+  def save(file)
+    File.open(file, 'w') do |file|
+      Zlib::GzipWriter.wrap(file) do |file|
+        file.puts DB_FORMAT
+        file.puts Oj.dump @paths
+        file.puts Oj.dump @files
+        file.puts Oj.dump @tables
+      end
+    end
+  end
+
+  def add_paths(paths)
+    paths -= @paths
+    return if paths.empty?
+    @paths += paths
+    files = paths.map {|p| self.class.files_from_path(p)}.flatten
+    return if files.empty?
+    if @progress
+      pbar = ProgressBar.create(title: "Building", total: files.length, format: PBAR_FORMAT, length: 80)
+    end
+    files.each do |f|
+      add_file(f)
+      pbar.increment if @progress
+    end
+  end
+
+  def update
+    new_files = (@paths.map {|p| self.class.files_from_path(p)}.flatten) - @files.keys
+    if @progress
+      pbar = ProgressBar.create(title: "Updating", total: new_files.length + @files.length, format: PBAR_FORMAT, length: 80)
+    end
+    changed = @files.keys.map do |f|
+      changed = update_file(f)
+      pbar.increment if @progress
+      changed
+    end
+    new_files.each do |f|
+      add_file(f)
+      pbar.increment if @progress
+    end
+    changed.any? || !new_files.empty?
+  end
+
+  def summary
+    ret = {}
+
+    @tables.each_key do |key|
+      ret[key] = @tables[key].keys.count
+    end
+
+    ret
+  end
+
+  private
+
+  def self.files_from_path(path)
+    if File.file?(path)
+      [path]
+    elsif File.directory?(path)
+      Dir[File.join(path, "**", "*")].select {|p| File.file?(p)}
+    else
+      []
+    end
+  end
+
+  def db_by_line()
+    tmpdb = {}
+    @tables.each do |tbl, vals|
+      vals.each do |key, val|
+        val.each do |entry|
+          if entry[:line_no]
+            tmpdb[entry[:file]] ||= {}
+            tmpdb[entry[:file]][entry[:line_no]] ||= []
+            tmpdb[entry[:file]][entry[:line_no]] << {tbl: tbl, key: key, entry: entry}
+          end
+        end
+      end
+    end
+    return tmpdb
+  end
+
+  def add_file(file)
+    return if not File.file? file
+
+    @files[file] = File.mtime(file).to_s
+
+    LANGS.each do |lang|
+      next if not lang.match_file file
+      lang.extract file do |tbl, key, args|
+        key = key.to_sym
+        @tables[tbl] ||= {}
+        @tables[tbl][key] ||= []
+        @tables[tbl][key] << StarScope::Datum.build(file, key, args)
+      end
+    end
+  end
+
+  def remove_file(file)
+    @files.delete(file)
+    @tables.each do |name, tbl|
+      tbl.each do |key, val|
+        val.delete_if {|dat| dat[:file] == file}
+      end
+    end
+  end
+
+  def update_file(file)
+    if not File.exists?(file) or not File.file?(file)
+      remove_file(file)
+      true
+    elsif DateTime.parse(@files[file]).to_time < File.mtime(file)
+      remove_file(file)
+      add_file(file)
+      true
+    else
+      false
+    end
+  end
+
+end

data/test/lib/test_db.rb

@@ -0,0 +1,44 @@
+require_relative '../test_helper'
+
+describe StarScope::DB do
+
+  before do
+    @db = StarScope::DB.new(false)
+  end
+
+  it "must raise on invalid tables" do
+    proc {@db.dump_table(:foo)}.must_raise StarScope::DB::NoTableError
+  end
+
+  it "must correctly add paths" do
+    paths = [GOLANG_SAMPLE, 'test/files']
+    @db.add_paths(paths)
+    @db.instance_eval('@paths').must_equal paths
+    @db.instance_eval('@files').keys.must_include GOLANG_SAMPLE
+    @db.instance_eval('@files').keys.must_include RUBY_SAMPLE
+  end
+
+  it "must correctly pick up new files in old paths" do
+    @db.instance_eval('@paths = ["test/files"]')
+    @db.update
+    files = @db.instance_eval('@files').keys
+    files.must_include GOLANG_SAMPLE
+    files.must_include RUBY_SAMPLE
+  end
+
+  it "must correctly remove old files in existing paths" do
+    @db.instance_eval('@paths = ["test/files"]')
+    @db.instance_eval('@files = {"test/files/foo"=>"2012-01-01"}')
+    @db.instance_eval('@files').keys.must_include 'test/files/foo'
+    @db.update
+    @db.instance_eval('@files').keys.wont_include 'test/files/foo'
+  end
+
+  it "must correctly load an old DB file" do
+    @db.load('test/files/db_old.json.gz')
+    @db.instance_eval('@paths').must_equal ['test/files']
+    @db.instance_eval('@files').keys.must_include GOLANG_SAMPLE
+    @db.instance_eval('@files').keys.must_include RUBY_SAMPLE
+  end
+
+end

data/test/lib/test_golang.rb

@@ -0,0 +1,67 @@
+require_relative '../test_helper'
+
+class TestGolang < MiniTest::Unit::TestCase
+  def setup
+    @db = {}
+    StarScope::Lang::Go.extract(GOLANG_SAMPLE) do |tbl, key, args|
+      key = key.to_sym
+      @db[tbl] ||= {}
+      @db[tbl][key] ||= []
+      @db[tbl][key] << args
+    end
+  end
+
+  def test_recognition
+    assert StarScope::Lang::Go.match_file(GOLANG_SAMPLE)
+    refute StarScope::Lang::Go.match_file('something_else')
+  end
+
+  def test_defs
+    assert @db.keys.include? :defs
+    defs = @db[:defs].keys
+    assert defs.include? :a
+    assert defs.include? :b
+    assert defs.include? :c
+    assert defs.include? :ttt
+    assert defs.include? :main
+    assert defs.include? :v1
+    assert defs.include? :v2
+    assert defs.include? :Sunday
+    assert defs.include? :Monday
+  end
+
+  def test_function_ends
+    assert @db.keys.include? :end
+    ends = @db[:end]
+    assert ends.keys.count == 1
+    assert ends.values.first.count == 5
+  end
+
+  def test_function_calls
+    assert @db.keys.include? :calls
+    calls = @db[:calls]
+    assert calls.keys.include? :a
+    assert calls.keys.include? :b
+    assert calls.keys.include? :c
+    assert calls.keys.include? :ttt
+    assert calls[:a].count == 3
+    assert calls[:b].count == 4
+    assert calls[:c].count == 2
+    assert calls[:ttt].count == 2
+  end
+
+  def test_variable_assigns
+    assert @db.keys.include? :assigns
+    assigns = @db[:assigns]
+    assert assigns.keys.include? :x
+    assert assigns.keys.include? :y
+    assert assigns.keys.include? :z
+    assert assigns.keys.include? :n
+    assert assigns.keys.include? :m
+    assert assigns[:x].count == 2
+    assert assigns[:y].count == 1
+    assert assigns[:z].count == 1
+    assert assigns[:n].count == 1
+    assert assigns[:m].count == 2
+  end
+end

data/test/lib/test_ruby.rb

@@ -0,0 +1,54 @@
+require_relative '../test_helper'
+
+class TestRuby < MiniTest::Unit::TestCase
+  def setup
+    @db = {}
+    StarScope::Lang::Ruby.extract(RUBY_SAMPLE) do |tbl, key, args|
+      key = key.to_sym
+      @db[tbl] ||= {}
+      @db[tbl][key] ||= []
+      @db[tbl][key] << args
+    end
+  end
+
+  def test_recognition
+    assert StarScope::Lang::Ruby.match_file(RUBY_SAMPLE)
+    assert StarScope::Lang::Ruby.match_file('bin/starscope')
+    refute StarScope::Lang::Ruby.match_file('test/files/sample_golang.go')
+  end
+
+  def test_function_defs
+    assert @db.keys.include? :defs
+    defs = @db[:defs].keys
+    assert defs.include? :DB
+    assert defs.include? :NoTableError
+    assert defs.include? :load
+    assert defs.include? :update
+    assert defs.include? :files_from_path
+  end
+
+  def test_function_ends
+    assert @db.keys.include? :end
+    ends = @db[:end]
+    assert ends.keys.count == 1
+    assert ends.values.first.count == 13
+  end
+
+  def test_function_calls
+    assert @db.keys.include? :calls
+    calls = @db[:calls]
+    assert calls.keys.include? :add_file
+    assert calls.keys.include? :each
+    assert calls[:add_file].count == 3
+    assert calls[:each].count == 8
+  end
+
+  def test_variable_assigns
+    assert @db.keys.include? :assigns
+    assigns = @db[:assigns]
+    assert assigns.keys.include? :pbar
+    assert assigns.keys.include? :PBAR_FORMAT
+    assert assigns[:pbar].count == 2
+    assert assigns[:PBAR_FORMAT].count == 1
+  end
+end

data/test/lib/test_version.rb

@@ -0,0 +1,9 @@
+require_relative '../test_helper'
+
+describe StarScope do
+
+  it "must be defined" do
+    StarScope::VERSION.wont_be_nil
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,6 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require File.expand_path('../../lib/starscope.rb', __FILE__)
+
+GOLANG_SAMPLE = 'test/files/sample_golang.go'
+RUBY_SAMPLE = 'test/files/sample_ruby.rb'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: starscope
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-10 00:00:00.000000000 Z
+date: 2014-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -131,13 +131,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
-- TODO.md
 - bin/starscope
 - lib/starscope.rb
 - lib/starscope/datum.rb
@@ -146,7 +146,14 @@ files:
 - lib/starscope/langs/ruby.rb
 - lib/starscope/version.rb
 - starscope.gemspec
-- test/test.go
+- test/files/db_old.json.gz
+- test/files/sample_golang.go
+- test/files/sample_ruby.rb
+- test/lib/test_db.rb
+- test/lib/test_golang.rb
+- test/lib/test_ruby.rb
+- test/lib/test_version.rb
+- test/test_helper.rb
 homepage: https://github.com/eapache/starscope
 licenses:
 - MIT
@@ -172,4 +179,12 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: A code indexer and analyzer
-test_files: []
+test_files:
+- test/files/db_old.json.gz
+- test/files/sample_golang.go
+- test/files/sample_ruby.rb
+- test/lib/test_db.rb
+- test/lib/test_golang.rb
+- test/lib/test_ruby.rb
+- test/lib/test_version.rb
+- test/test_helper.rb

data/TODO.md

@@ -1,6 +0,0 @@
-TODO
-=========
-
-* Parallelization, perhaps via https://github.com/grosser/parallel
-* Fuzzy matching, perhaps via https://github.com/seamusabshere/fuzzy_match
-* Export advanced ctags annotations