starscope 1.0.4 → 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9f58bc7549fa2f4d8d7da61d2771a1e6f75939e2
+  data.tar.gz: 1452b9af5221dc6bc2147d0149870e564e2344fc
+SHA512:
+  metadata.gz: d9c15bef6cef5e856d023944439efac3a0caa9c2b55f1b018e50d8f7c4fa6f2ced00b8bb595639958547426db52ae23abb9487611ff883638950dbac9876da48
+  data.tar.gz: 821ec54d44933500bffc87b871410fbd08cdbc3b9c2e9c0f28a9c4dadb289d07fd12215993763e0982b26013d82e1fd589a1cb3eba876bd852e958c495c17df5

data/.gitignore

@@ -16,4 +16,3 @@ tmp
 # YARD artifacts
 .yardoc
 _yardoc
-doc/

data/CHANGELOG.md

@@ -1,7 +1,30 @@
 Changelog
 =========
 
-v1.0.4 (trunk)
+v1.1.0 (trunk)
+--------------------
+
+Bug Fixes:
+ * Fixed cscope export when the string for a token appeared multiple times on
+   the same line, for example the function call `go("go")`.
+ * Fixed cscope export of function calls made from the global scope.
+ * Fixed cscope export of functions that end in punctuation (e.g. `include?`)
+ * Fixed golang parsing of global function calls.
+ * Fixed crash in line-mode when pressing `<return>` with no input.
+
+Improvements:
+ * Optimized and refactored database update path. Now much simpler and about 8%
+   faster.
+ * Documented the database format and language API. Added a user guide which is
+   much more complete than the simpler output of the `--help` flag.
+
+Misc:
+ * Rename --no-progress to --quiet, and make sure all operations provide some
+   indication of success/failure except when quieted.
+ * Dynamically load language extractors, so new ones can be dropped in with no
+   other code changes.
+
+v1.0.4 (2014-06-10)
 --------------------
 
 Improvements:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    starscope (1.0.4)
+    starscope (1.1.0)
       oj (~> 2.9)
       parser (~> 2.1)
       ruby-progressbar (~> 1.5)
@@ -12,14 +12,14 @@ GEM
     ast (2.0.0)
     coderay (1.1.0)
     method_source (0.8.2)
-    minitest (5.3.4)
-    oj (2.9.4)
+    minitest (5.4.0)
+    oj (2.9.9)
     parser (2.1.9)
       ast (>= 1.1, < 3.0)
       slop (~> 3.4, >= 3.4.5)
-    pry (0.9.12.6)
-      coderay (~> 1.0)
-      method_source (~> 0.8)
+    pry (0.10.0)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
       slop (~> 3.4)
     rake (10.3.2)
     ruby-progressbar (1.5.1)

data/README.md

@@ -37,6 +37,14 @@ $ starscope -e cscope
 
 More flags and options are available, run `starscope --help` for the complete list.
 
+More Documentation
+------------------
+
+ * [User Guide](doc/USER_GUIDE.md)
+ * [Database Format](doc/DB_FORMAT.md)
+ * [Language Support](doc/LANGUAGE_SUPPORT.md)
+ * [Version History](CHANGELOG.md)
+
 Other Uses
 ----------
 

data/Rakefile

@@ -2,8 +2,7 @@ require 'bundler/gem_tasks'
 require 'rake/testtask'
 
 Rake::TestTask.new do |t|
-  t.libs << 'lib/starscope'
-  t.test_files = FileList['test/lib/test_*.rb']
+  t.test_files = FileList['test/unit/test_*.rb', 'test/functional/test_*.rb']
 end
 
 desc 'Run tests'

data/bin/starscope

@@ -11,11 +11,10 @@ DEFAULT_DB='.starscope.db'
 DEFAULT_CTAGS='tags'
 DEFAULT_CSCOPE='cscope.out'
 
-options = {:show_progress => true,
-           :read => true,
+options = {:read => true,
            :write => true,
            :update => true,
-           :verbose => false,
+           :output => :normal,
            :db => DEFAULT_DB
 }
 
@@ -31,6 +30,9 @@ database by recursing in the current directory.
 
 Scoped queries must use `::` as the scope separator, even for languages which
 have their own scope syntax.
+
+Website: https://github.com/eapache/starscope
+User Manual: https://github.com/eapache/starscope/blob/master/doc/USER_GUIDE.md
 END
 
   opts.separator "\nQueries"
@@ -73,11 +75,19 @@ END
     puts StarScope::VERSION
     exit
   end
-  opts.on("--verbose", "Print extra details") do
-    options[:verbose] = true
+  opts.on("--verbose", "Print extra status messages") do
+    if options[:output] == :quiet
+      $stderr.puts "Can't be both verbose and quiet"
+      exit(1)
+    end
+    options[:output] = :verbose
   end
-  opts.on("--no-progress", "Don't show progress-bar when updating DB") do
-    options[:show_progress] = false
+  opts.on("--quiet", "Print fewer messages") do
+    if options[:output] == :verbose
+      $stderr.puts "Can't be both verbose and quiet"
+      exit(1)
+    end
+    options[:output] = :quiet
   end
 
   opts.separator <<END
@@ -90,7 +100,9 @@ END
 end.parse!
 
 def print_summary(db)
-  db.summary.each do |name, count|
+  summary = db.summary
+  puts "No tables" if summary.empty?
+  summary.each do |name, count|
     printf("%-8s %5d records\n", name, count)
   end
 end
@@ -101,12 +113,12 @@ def run_query(db, table, value)
     return false
   end
   results = db.query(table.to_sym, value)
-  if results
+  if !results || results.empty?
+    puts "No results found."
+  else
     results.sort_by {|x| x[:name].join(' ')}.each do |rec|
       puts StarScope::Record.format(rec)
     end
-  else
-    puts "No results found."
   end
   return true
 rescue StarScope::DB::NoTableError
@@ -128,23 +140,28 @@ rescue StarScope::DB::NoTableError
   return false
 end
 
-def export(db, param)
+def export(db, param, output)
   format, path = param.split(',', 2)
+
   case format
-  when 'ctags'
-    File.open(path || DEFAULT_CTAGS, 'w') do |file|
-      db.export_ctags(file)
-    end
-  when 'cscope'
-    File.open(path || DEFAULT_CSCOPE, 'w') do |file|
-      db.export_cscope(file)
-    end
+  when 'ctags'; path ||= DEFAULT_CTAGS
+  when 'cscope'; path ||= DEFAULT_CSCOPE
   else
-    puts "Unrecognized export format"
+    $stderr.puts "Unrecognized export format \"#{format}\""
+    return
+  end
+
+  print "Exporting to '#{path}' in format '#{format}'... " unless output == :quiet
+  File.open(path, 'w') do |file|
+    case format
+    when 'ctags'; db.export_ctags(file)
+    when 'cscope'; db.export_cscope(file)
+    end
   end
+  puts "done" unless output == :quiet
 end
 
-db = StarScope::DB.new(options[:show_progress], options[:verbose])
+db = StarScope::DB.new(options[:output])
 
 db_exists = File.exists?(options[:db])
 
@@ -176,7 +193,7 @@ new_data ||= updated
 
 db.save(options[:db]) if options[:write] and (new_data or not db_exists)
 
-export(db, options[:export]) if options[:export]
+export(db, options[:export], options[:output]) if options[:export]
 
 if options[:query]
   table, query = options[:query].split(',', 2)
@@ -212,6 +229,7 @@ if options[:linemode]
   puts "Run your query as 'TABLE QUERY' or run '!help' for more information."
   begin
     while input = Readline.readline("> ", true)
+      next if input.empty?
       cmd, param = input.split(' ', 2)
       if cmd[0] == '!'
         case cmd[1..-1]
@@ -219,7 +237,7 @@ if options[:linemode]
           dump(db, param)
         when "export"
           if param
-            export(db, param)
+            export(db, param, :normal)
           else
             puts "!export requires an argument"
           end

data/doc/DB_FORMAT.md

@@ -0,0 +1,42 @@
+Database Format
+===============
+
+The StarScope database format has gone through a couple of iterations over the
+last year, but the current one is pretty nice and I don't see it changing again
+in the near future. We magically read old formats and convert them, so we only
+document the current version here.
+
+The basic format of the database is [gzipped](https://en.wikipedia.org/wiki/Gzip)
+text. Gzip is a common compression algorithm; most languages include support for
+it by default, and it is readable on effectively every platform. Underneath the
+compression, the file has three lines:
+ * version number
+ * metadata
+ * databases
+
+Version Number
+--------------
+
+This is just the ascii character '5'. If the format changes signficantly, it
+will be increased, but that is unlikely.
+
+Metadata
+--------
+
+This is a [JSON](https://en.wikipedia.org/wiki/Json) object. Keys include:
+ * `:paths` - the paths containing the files to scan
+ * `:excludes` - the paths and patterns to exclude from scanning
+ * `:files` - the files previously scanned (including things like last-modified
+   time)
+ * `:version` - The StarScope version which wrote the database (not the same as
+   the database format version).
+
+Databases
+---------
+
+This is also a [JSON](https://en.wikipedia.org/wiki/Json) object. The keys are
+table names, and the values are the tables themselves. Each table is an array of
+records, and each record is itself a JSON object. The only keys that are
+guaranteed to exist in every record are `:file` (the name of the file the record
+is from) and `:name` (an array containing the individual components of the
+fully-scoped name).

data/doc/LANGUAGE_SUPPORT.md

@@ -0,0 +1,80 @@
+Language Support
+================
+
+Already Supported
+-----------------
+
+ * [Ruby](https://www.ruby-lang.org/)
+ * [Golang](http://golang.org/)
+
+How to Add Another Language
+---------------------------
+
+Adding support for a new language is really easy, and pretty generic - all of
+the fancy features like cscope export should "just work" since they don't depend
+on anything language-specific, they just read the internal database record
+format.
+
+For this doc, we're going to pretend to add support for a language called
+"MyLanguage". Create a file called `mylanguage.rb` in `lib/starscope/langs/` and
+drop the following template in:
+
+```ruby
+module StarScope::Lang
+  module Mylanguage
+
+    def self.match_file(name)
+      name.end_with?(".mylang")
+    end
+
+    def self.extract(file)
+      # TODO
+    end
+
+  end
+end
+```
+
+This code is pretty simple: we define a module called
+`StarScope::Lang::Mylanguage` and give it two public module methods:
+ * `match_file` takes the name of the file and returns a boolean if that file is
+   written in MyLanguage or not. This can be as simple as checking the file
+   extension (which the sample code does) or looking for a shell #! line, or
+   anything you want.
+ * `extract` takes a readable file handle pointing to the file, and must parse
+   the file, `yield`ing records as it finds function definitions and the like.
+
+The record requirements are pretty straight-forward:
+```ruby
+yield table, name, extras={}
+```
+The first yielded argument must be the symbol of the table in which the record
+belongs (basically its type). Current tables already include:
+ * `calls` for function calls
+ * `defs` for definitions of classes, functions, modules, etc.
+ * `end` for the matching *end* of each definition
+ * `assigns` for variable assignment
+ * `requires` for required files in Ruby
+ * `imports` for imported packages in Golang
+
+Try to use pre-existing tables where possible, but feel free to add more if the
+language has some weird feature that doesn't map to any of the above. You don't
+have to do anything special to create a new table, just yield the appropriate
+symbol.
+
+The second yielded argument is the name (string or symbol) of the token that
+you want to add: the name of the function being called, or the name of the class
+being defined, etc. If the name is scoped (such as "def MyModule::MyClass") pass
+an array `["MyModule", "MyClass"]`.
+
+Finally, the entirely-optional `extras` is a hash of whatever other extra values
+you want. Some existing ones that you may want to use include:
+ * `line_no` for the line (starting at 1) where the record occurs - this is
+   necessary for cscope and ctags export
+ * `col` for the column in the line where the name occurs
+ * `type` for the type of definition (`:func`, `:class`, etc)
+
+And that's it! Parse your files, yield your records, and the StarScope engine
+takes care of everything else for you. If you've added support for a language
+that you think others might find useful, please contribute it (with tests!) via
+pull request.

data/doc/USER_GUIDE.md

@@ -0,0 +1,135 @@
+StarScope User Guide
+====================
+
+
+Installation
+------------
+
+StarScope is a ruby gem available at https://rubygems.org/gems/starscope.
+Install it with:
+```
+$ gem install starscope
+```
+
+This should place a program called `starscope` in your path.
+
+Quick Start
+-----------
+
+Build your database by just running it in the project directory:
+```
+$ cd ~/my-project
+$ starscope
+```
+
+Ask it things directly:
+```
+$ starscope -q calls,new # Lists all callers of new
+```
+
+Export it to various formats for use with your editor:
+```
+$ starscope -e ctags
+$ starscope -e cscope
+```
+
+Database Options
+----------------
+
+The default database is `.starscope.db` in the current directory. If you want
+to use another file, specify one with `-f` or `--file`.
+
+The default behaviour is always to read the database (if it exists), update it,
+and write out the updated version. You can control this behaviour by passing any
+of the `--no-read`, `--no-write` and `--no-update` flags.
+
+To get a summary of the current database contents, pass the `-s` or `--summary`
+flag.
+
+Paths
+-----
+
+StarScope has powerful options with sane defaults for managing which paths get
+scanned for files and which do not. By default when creating a new database,
+StarScope will scan all files recursively in the current directory. To scan
+specific paths or files instead, pass them as arguments (so `starscope myfolder`
+would only scan files in `myfolder/`).
+
+Paths are saved in the database metadata - once you have created a database with
+custom paths, all subsequent operations will remember and use those paths even
+when they are not explicitly specified. At any time, specifying *more* paths
+will add them (and the files they contain) to the database without removing any
+existing paths.
+
+You can also exclude certain paths from processing by passing the `-x` or
+`--exclude` flag with the desired pattern. In a new project, `starscope
+--exclude test/` will scan all files *except* the ones in the `test/` directory.
+Excluded patterns are also remembered, and can be added at any time. If an
+existing file in the database matches a newly added exclusion rule, it will be
+removed.
+
+Queries
+-------
+
+To query the starscope database, pass the `-q` or `--query` flag with an
+argument in the following format: `TABLE,QUERY`. For example, `-q calls,new`
+would list all callers of `new` and `-q defs,bar` would list places that define
+a method or class named `bar`. See the [language support
+documentation](LANGUAGE_SUPPORT.md) for a list of the most common tables, or use
+the `--summary` flag to list all the tables in the current database.
+
+You can also search for scoped names such as `MyClass.new`. To do this, you must
+specify the scope with `::`, even if the language or instance you are searching
+for uses another character like a dot. So, for example, `-q calls,MyClass::new`.
+
+Queries using regular expressions are generally supported, and will be tried
+against both the base name and the fully-qualified name (again using `::` as the
+scope separator).
+
+Exporting
+---------
+
+StarScope can export its database into two other formats for use with
+third-party tools:
+ * cscope format (default path: `cscope.out`)
+ * ctags format (default path: `tags`)
+
+To export, pass the `-e` or `--export` flag with one of the above formats. Each
+format has its own default path which is where the exported file will go. If you
+want to specify a custom location, append it after the format with a comma (so
+`--export cscope,myfile` would export to `myfile` in the cscope format).
+
+You can also dump entire tables (or the entire database) to raw text output for
+manual inspection. This is mostly useful for debugging, but means you can pipe
+it to sed (for example) if you wanted to do something fancy. This can be done
+with the `-d` or `--dump` flag, which takes an optional argument of which table
+to dump (if no table is specified, it dumps all tables).
+
+Line-Mode
+---------
+
+Specifying `-l` or `--line-mode` places you into line-oriented mode, letting you
+run multiple queries without reloading the database each time. In line mode,
+input is normally a query of the form `TABLE QUERY`, or a special command
+starting with a `!`. Recognized special commands generally map to non-line-mode
+options:
+ * `!dump [TABLE]` - same as the `--dump` flag
+ * `!export FORMAT[,PATH]` - same as the `--export` flag
+ * `!summary` - same as the `--summary` flag
+ * `!update` - updates the database without exiting line-mode
+ * `!help` - prints basic line-mode help
+ * `!version` - same as the `--version` flag
+ * `!quit` - exit line-mode
+
+
+Miscellaneous
+-------------
+
+Pass `-h` or `--help` to get a brief summary of the availble options.
+
+Pass `-v` or `--version` to get the current version number.
+
+Pass `--verbose` to get a great deal more output about what it is doing.
+
+Pass `--quiet` to remove all non-critical output (you will still get error
+messages, query results, etc).