starscope 1.5.7 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 16cc0b7d5c3226ca3680b475dfcd923348352d9f
-  data.tar.gz: f3c98060b7b84c54fc13fab3469387f691dfb76f
+SHA256:
+  metadata.gz: d71f7d9d5748e9df63987f08669e56866a5669a7877a5b78bec7274ae6b11e3b
+  data.tar.gz: a037cc0b16644f69a37c1d9a00f076a036353cb84cbcaef39e17f864af2c70eb
 SHA512:
-  metadata.gz: 6b56db7031396663009a1d062b7444ada17e3d103f2bb972e6bd94b6211eb22a4a44d3891ae1e9a01dc8317800d64438c3f45d4fa25fc8dd17e8e8e1f9ce36bc
-  data.tar.gz: 7080292f2aaa289b20733edf0a40594ad213864cdda3b7c014143aae6405c51aa142993b52058313f8ddfa0703401db21c47eec0b7d30cfcc36de94648c344b4
+  metadata.gz: 85d4390356102ec0dfc658fa99bb8816a197f48f990abe56fd5b1df7ac85d3d1ffdd93666ca463ca9c42a362ae81a369cc81fa700c83729a9e73a9ba1e431730
+  data.tar.gz: 6cb4cc84d760ba1e5114f2401878e9312eff5e1b7d30ae7b3c4feec10a76c0f87f1f3d00de59417aee8e5e57aa2eb9744ae6a851303b964d9b8edaf8fcec3dad

data/.github/workflows/ruby-ci.yml

@@ -0,0 +1,26 @@
+name: Ruby CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0', '3.1']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake test
+    - name: Lint
+      run: bundle exec rubocop

data/.rubocop.yml

@@ -1,6 +1,15 @@
+require:
+  - rubocop-minitest
+  - rubocop-rake
+
 AllCops:
+  NewCops: enable
   Exclude:
-    - 'test/fixtures/sample_ruby.rb'
+    - test/fixtures/sample_ruby.rb
+    - vendor/bundle/**/*
+
+Layout/LineLength:
+  Max: 120
 
 Metrics/AbcSize:
   Enabled: false
@@ -17,9 +26,6 @@ Metrics/ClassLength:
 Metrics/CyclomaticComplexity:
   Enabled: false
 
-Metrics/LineLength:
-  Max: 120
-
 Metrics/MethodLength:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,6 +1,24 @@
 Changelog
 =========
 
+v1.6.0 (2022-02-10)
+-------------------
+
+Improvements:
+ * Support per-directory `.starscope.json` config (#181).
+ * Support abbreviated special commands in line mode (e.g. `!u`, `!s`, `!q`).
+ * Properly support non-ASCII identifiers in Golang.
+ * Tiny performance improvements.
+
+Bug Fixes:
+ * Strip non-ASCII characters from cscope export to avoid cscope crashes (#182).
+
+Misc:
+ * Drop support for ancient Marshall-format databases.
+ * Drop support for several old Ruby versions. 2.6 is now the oldest supported.
+ * Dependency upgrades.
+ * Documentation improvements.
+
 v1.5.7 (2019-04-04)
 --------------------
 

data/README.md

@@ -2,7 +2,7 @@ Starscope
 =========
 
 [![Gem Version](https://img.shields.io/gem/v/starscope.svg)](https://rubygems.org/gems/starscope)
-[![Build Status](https://travis-ci.org/eapache/starscope.svg?branch=master)](https://travis-ci.org/eapache/starscope)
+[![Ruby CI](https://github.com/eapache/starscope/actions/workflows/ruby-ci.yml/badge.svg)](https://github.com/eapache/starscope/actions/workflows/ruby-ci.yml)
 [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-active-blue.svg)](https://eapache.github.io/conduct.html)
 
 Starscope is a code indexer, search and navigation tool for

data/Rakefile

@@ -6,7 +6,10 @@ Rake::TestTask.new do |t|
   t.test_files = FileList['test/**/*_test.rb']
 end
 
-RuboCop::RakeTask.new
+RuboCop::RakeTask.new do |t|
+  t.requires << 'rubocop-minitest'
+  t.requires << 'rubocop-rake'
+end
 
 desc 'Run tests and style checks'
-task default: [:test, :rubocop]
+task default: %i[test rubocop]

data/bin/starscope

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-lib = File.expand_path('../../lib', __FILE__)
+lib = File.expand_path('../lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'oj'
@@ -9,7 +9,7 @@ require 'readline'
 require 'starscope'
 
 DEFAULT_DB = '.starscope.db'.freeze
-CONFIG_FILE = File.join(Dir.home, '.starscope.json')
+CONFIG_FILE = File.exist?('.starscope.json') ? '.starscope.json' : File.join(Dir.home, '.starscope.json')
 GLOBAL_CONFIG = File.exist?(CONFIG_FILE) ? Oj.load_file(CONFIG_FILE, symbol_keys: true) : {}
 
 options = { read: true,
@@ -21,20 +21,20 @@ options = { read: true,
 
 # Options Parsing
 OptionParser.new do |opts|
-  opts.banner = <<END
-Usage: starscope [options] [PATHS]
+  opts.banner = <<~BANNER
+    Usage: starscope [options] [PATHS]
 
-The default database is `#{DEFAULT_DB}` if you don't specify one with -f.
-The default behaviour is to read and update the database.
-If no database exists and no PATHS are specified, Starscope builds a new
-database by recursing in the current directory.
+    The default database is `#{DEFAULT_DB}` if you don't specify one with -f.
+    The default behaviour is to read and update the database.
+    If no database exists and no PATHS are specified, Starscope builds a new
+    database by recursing in the current directory.
 
-Scoped queries must use `::` as the scope separator, even for languages which
-have their own scope syntax.
+    Scoped queries must use `::` as the scope separator, even for languages which
+    have their own scope syntax.
 
-Website: https://github.com/eapache/starscope
-User Manual: https://github.com/eapache/starscope/blob/master/doc/USER_GUIDE.md
-END
+    Website: https://github.com/eapache/starscope
+    User Manual: https://github.com/eapache/starscope/blob/master/doc/USER_GUIDE.md
+  BANNER
 
   opts.separator "\nQueries"
   opts.on('-d', '--dump [TABLE]', 'Dumps the DB or specified table to stdout') do |tbl|
@@ -81,32 +81,32 @@ END
   end
   opts.on('--verbose', 'Print extra status messages') do
     if options[:output] == :quiet
-      $stderr.puts "Can't be both verbose and quiet"
+      warn "Can't be both verbose and quiet"
       exit(1)
     end
     options[:output] = :verbose
   end
   opts.on('--quiet', 'Print fewer messages') do
     if options[:output] == :verbose
-      $stderr.puts "Can't be both verbose and quiet"
+      warn "Can't be both verbose and quiet"
       exit(1)
     end
     options[:output] = :quiet
   end
 
-  opts.separator <<END
-\nEXPORTING
-    At the moment two export formats are supported: 'ctags' and 'cscope'. If
-    you don't specify a path, the output is written to the file '#{Starscope::Exportable::CTAGS_DEFAULT_PATH}' (for
-    ctags) or '#{Starscope::Exportable::CSCOPE_DEFAULT_PATH}' (for cscope) in the current directory.
-END
+  opts.separator <<~TAIL
+    \nEXPORTING
+        At the moment two export formats are supported: 'ctags' and 'cscope'. If
+        you don't specify a path, the output is written to the file '#{Starscope::Exportable::CTAGS_DEFAULT_PATH}' (for
+        ctags) or '#{Starscope::Exportable::CSCOPE_DEFAULT_PATH}' (for cscope) in the current directory.
+  TAIL
 end.parse!
 
 def print_summary(db)
   tables = db.tables
   puts 'No tables' if tables.empty?
   tables.sort.each do |table|
-    printf("%-9s %6d records\n", table, db.records(table).length)
+    printf("%<table>-9s %<length>6d records\n", table: table, length: db.records(table).length)
   end
 end
 
@@ -134,7 +134,7 @@ def run_query(db, query, separator)
   match = match.join(separator)
 
   if match.empty?
-    $stderr.puts 'Invalid input - no query found.'
+    warn 'Invalid input - no query found.'
     return false
   end
   tables = (table == '*' ? db.tables : table.to_sym)
@@ -146,10 +146,10 @@ def run_query(db, query, separator)
       puts format_record(db, rec)
     end
   end
-  return true
+  true
 rescue Starscope::DB::NoTableError
-  $stderr.puts "Table '#{table}' doesn't exist."
-  return false
+  warn "Table '#{table}' doesn't exist."
+  false
 end
 
 def dump_table(db, table)
@@ -174,22 +174,22 @@ def dump(db, table)
   when '_files'
     puts db.metadata(:files).keys
   when /^_/
-    puts db.metadata(table[1..-1].to_sym)
+    puts db.metadata(table[1..].to_sym)
   else
     dump_table(db, table.to_sym)
   end
 
-  return true
+  true
 rescue Starscope::DB::NoTableError
-  $stderr.puts "Table '#{table}' doesn't exist."
-  return false
+  warn "Table '#{table}' doesn't exist."
+  false
 end
 
 def export(db, param)
   format, path = param.split(',', 2)
   db.export(format.to_sym, path)
 rescue Starscope::Exportable::UnknownExportFormatError
-  $stderr.puts "Unrecognized export format \"#{format}\""
+  warn "Unrecognized export format \"#{format}\""
 end
 
 output = Starscope::Output.new(options[:output])
@@ -246,18 +246,19 @@ if options[:dump]
 end
 
 def linemode_help
-  <<END
-Input can be a query of the form 'TABLE QUERY' or a special command starting
-with a '!'. Recognized special commands are:
-  !dump [TABLE]
-  !export FORMAT[,PATH]
-  !summary
-  !update
-
-  !help
-  !version
-  !quit
-END
+  <<~HELP
+    Input can be a query of the form 'TABLE QUERY' or a special command starting
+    with a '!'. Recognized special commands are:
+      !dump [TABLE]
+      !export FORMAT[,PATH]
+      !summary
+      !update
+
+      !help
+      !version
+      !quit
+    Abbreviations of special commands (e.g `!u` or `!q`) are also recognized.
+  HELP
 end
 
 if options[:linemode]
@@ -265,27 +266,28 @@ if options[:linemode]
   begin
     while (input = Readline.readline('> ', true))
       next if input.empty?
+
       cmd, param = input.split(' ', 2)
       if cmd[0] == '!'
-        case cmd[1..-1]
-        when 'dump'
+        case cmd[1..]
+        when 'dump', 'd'
           dump(db, param)
-        when 'export'
+        when 'export', 'e', 'ex'
           if param
             export(db, param)
           else
             puts '!export requires an argument'
           end
-        when 'summary'
+        when 'summary', 's', 'sum'
           print_summary(db)
-        when 'update'
+        when 'update', 'u', 'up'
           changed = db.update
           db.save(options[:db]) if options[:write] && changed
-        when 'help'
+        when 'help', 'h'
           puts linemode_help
-        when 'version'
+        when 'version', 'v'
           puts Starscope::VERSION
-        when 'quit'
+        when 'quit', 'q'
           exit
         else
           puts "Unknown command: '#{input}', try '!help'."

data/doc/LANGUAGE_SUPPORT.md

@@ -9,6 +9,14 @@ Already Supported
  * [JavaScript](https://en.wikipedia.org/wiki/JavaScript)
    (including ES6/ES7 and JSX via [Babel](https://babeljs.io/))
 
+External Support
+-----------------
+
+Other languages are sometimes supported in third-party plugin files, or forks
+which don't upstream their changes, but I try and list them here. I make no
+warranty that these are still maintained or even work.
+ * [Puppet](https://github.com/fihuer/starscope/tree/language/puppet)
+
 How to Add Another Language
 ---------------------------
 

data/doc/USER_GUIDE.md

@@ -109,14 +109,22 @@ Excluded patterns are also remembered, and can be added at any time. If an
 existing file in the database matches a newly added exclusion rule, it will be
 removed.
 
-For commonly excluded files you can create a `~/.starscope.json` file with
-contents like:
+Paths to add and exclude are recursive by default. If you need more control, you
+may use any special characters accepted by Ruby's
+[`File::fnmatch`](https://ruby-doc.org/core-2.6.9/File.html#method-c-fnmatch).
+
+You can automatically exclude files on a per-directory basis by creating a
+`.starscope.json` file in a given directory with contents like:
 ```json
 {
-  "excludes": ["foo", "bar"]
+  "excludes": ["foo", "bar/", "**/*.ext"]
 }
 ```
-Patterns listed there will be excluded from all starscope databases by default.
+
+For commonly excluded files you can create a home directory config file at 
+`~/.starscope.json`. Patterns listed there will be excluded from all starscope 
+databases by default. Currently `excludes` is the only valid key for
+`.starscope.json` files.
 
 Queries
 -------
@@ -177,6 +185,8 @@ non-line-mode options:
  * `!version` - same as the `--version` flag
  * `!quit` - exit line-mode
 
+Abbreviations of special commands are also recognized (e.g. `!q` for `!quit`,
+`!s` for `!summary`, etc).
 
 Miscellaneous
 -------------

data/lib/starscope/db.rb

@@ -17,7 +17,7 @@ module Starscope
     FRAGMENT = :'!fragment'
 
     # dynamically load all our language extractors
-    Dir.glob("#{File.dirname(__FILE__)}/langs/*.rb").each { |path| require path }
+    Dir.glob("#{File.dirname(__FILE__)}/langs/*.rb").sort.each { |path| require path }
 
     langs = {}
     extractors = []
@@ -31,6 +31,7 @@ module Starscope
 
     class NoTableError < StandardError; end
     class UnknownDBFormatError < StandardError; end
+    class TooOldDBFormatError < StandardError; end
 
     def initialize(output, config = {})
       @output = output
@@ -84,9 +85,9 @@ module Starscope
       paths = paths.map { |p| self.class.normalize_glob(p) }
       @meta[:paths] += paths
       @meta[:paths].uniq!
-      files = Dir.glob(paths).select { |f| File.file? f }
-      files.delete_if { |f| matches_exclude?(f) }
+      files = Dir.glob(paths).select { |f| File.file?(f) && !matches_exclude?(f) }
       return if files.empty?
+
       @output.new_pbar('Building', files.length)
       add_files(files)
       @output.finish_pbar
@@ -149,14 +150,12 @@ module Starscope
 
     def open_db(filename)
       File.open(filename, 'rb') do |file|
-        begin
-          Zlib::GzipReader.wrap(file) do |stream|
-            parse_db(stream)
-          end
-        rescue Zlib::GzipFile::Error
-          file.rewind
-          parse_db(file)
+        Zlib::GzipReader.wrap(file) do |stream|
+          parse_db(stream)
         end
+      rescue Zlib::GzipFile::Error
+        file.rewind
+        parse_db(file)
       end
     end
 
@@ -166,28 +165,26 @@ module Starscope
       when DB_FORMAT
         @meta   = Oj.load(stream.gets)
         @tables = Oj.load(stream.gets)
-        return true
+        true
       when 3..4
         # Old format, so read the directories segment then rebuild
         add_paths(Oj.load(stream.gets))
-        return false
+        false
       when 0..2
-        # Old format (pre-json), so read the directories segment then rebuild
-        len = stream.gets.to_i
-        add_paths(Marshal.load(stream.read(len)))
-        return false
+        raise TooOldDBFormatError
       else
         raise UnknownDBFormatError
       end
     rescue Oj::ParseError
       stream.rewind
       raise unless stream.gets.to_i == DB_FORMAT
+
       # try reading as formated json, which is much slower, but it is sometimes
       # useful to be able to directly read your db
       objects = []
       Oj.load(stream) { |obj| objects << obj }
       @meta, @tables = objects
-      return true
+      true
     end
 
     def fixup
@@ -233,7 +230,7 @@ module Starscope
       self.class.extractors.each do |extractor|
         begin
           next unless extractor.match_file file
-        rescue => e
+        rescue StandardError => e
           @output.normal("#{extractor} raised \"#{e}\" while matching #{file}")
           next
         end
@@ -271,8 +268,7 @@ module Starscope
         extract_file(Starscope::FragmentExtractor.new(lang, frags), file, line_cache, lines)
         @meta[:files][file][:sublangs] << lang
       end
-
-    rescue => e
+    rescue StandardError => e
       @output.normal("#{extractor} raised \"#{e}\" while extracting #{file}")
     ensure
       # metadata must be created for any record that was inserted into a tbl
@@ -280,9 +276,7 @@ module Starscope
       @meta[:files][file][:lang] = extractor.name.split('::').last.to_sym
       @meta[:files][file][:lines] = lines
 
-      if extractor_metadata.is_a? Hash
-        @meta[:files][file] = extractor_metadata.merge!(@meta[:files][file])
-      end
+      @meta[:files][file] = extractor_metadata.merge!(@meta[:files][file]) if extractor_metadata.is_a? Hash
     end
 
     def file_changed(name)
@@ -301,6 +295,7 @@ module Starscope
     def language_out_of_date(lang)
       return false unless lang
       return true unless LANGS[lang]
+
       (@meta[:langs][lang] || 0) < LANGS[lang]
     end
 
@@ -334,7 +329,8 @@ module Starscope
         args
       end
 
-      def extractors # so we can stub it in tests
+      # so we can stub it in tests
+      def extractors
         EXTRACTORS
       end
     end

data/lib/starscope/exportable.rb

@@ -4,6 +4,7 @@ module Starscope
   module Exportable
     CTAGS_DEFAULT_PATH = 'tags'.freeze
     CSCOPE_DEFAULT_PATH = 'cscope.out'.freeze
+    ASCII = Encoding.find('ASCII')
 
     class UnknownExportFormatError < StandardError; end
 
@@ -39,14 +40,14 @@ module Starscope
     private
 
     def export_ctags(file, path_prefix)
-      file.puts <<END
-!_TAG_FILE_FORMAT	2	/extended format/
-!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
-!_TAG_PROGRAM_AUTHOR	Evan Huus /eapache@gmail.com/
-!_TAG_PROGRAM_NAME	Starscope //
-!_TAG_PROGRAM_URL	https://github.com/eapache/starscope //
-!_TAG_PROGRAM_VERSION	#{Starscope::VERSION}	//
-END
+      file.puts <<~HEADER
+        !_TAG_FILE_FORMAT	2	/extended format/
+        !_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+        !_TAG_PROGRAM_AUTHOR	Evan Huus /eapache@gmail.com/
+        !_TAG_PROGRAM_NAME	Starscope //
+        !_TAG_PROGRAM_URL	https://github.com/eapache/starscope //
+        !_TAG_PROGRAM_VERSION	#{Starscope::VERSION}	//
+      HEADER
       defs = (@tables[:defs] || {}).sort_by { |x| x[:name][-1].to_s }
       defs.each do |record|
         file.puts ctag_line(record, @meta[:files][record[:file]], path_prefix)
@@ -150,7 +151,7 @@ END
       file.print("0\n")
       file.print("#{files.length}\n")
       buf = ''
-      files.each { |f| buf << f + "\n" }
+      files.each { |f| buf << "#{f}\n" }
       file.print("#{buf.length}\n#{buf}")
     end
 
@@ -159,6 +160,7 @@ END
       @tables.each do |tbl, records|
         records.each do |record|
           next unless record[:line_no]
+
           record[:tbl] = tbl
           db[record[:file]] ||= {}
           db[record[:file]][record[:line_no]] ||= []
@@ -177,16 +179,14 @@ END
         # use the column if we have it, otherwise fall back to scanning
         index = record[:col] || line.index(key)
 
-        while index && !valid_index?(line, index, key)
-          index = line.index(key, index + 1)
-        end
+        index = line.index(key, index + 1) while index && !valid_index?(line, index, key)
 
         next if index.nil?
 
         # Strip trailing non-word characters, otherwise cscope barfs on
         # function names like `include?`
         if key =~ /^\W*$/
-          next unless [:defs, :end].include?(record[:tbl])
+          next unless %i[defs end].include?(record[:tbl])
         else
           key.sub!(/\W+$/, '')
         end
@@ -209,25 +209,23 @@ END
 
         index = line.index(key, prev)
 
-        while index && index + key.length < offset && !valid_index?(line, index, key)
-          index = line.index(key, index + 1)
-        end
+        index = line.index(key, index + 1) while index && index + key.length < offset && !valid_index?(line, index, key)
 
         next unless index && index + key.length < offset
 
         buf << cscope_plaintext(line, prev, index) << "\n"
-        buf << "#{key}\n"
+        buf << "#{strip_unicode(key)}\n"
         prev = index + key.length
       end
 
       buf << cscope_plaintext(line, prev, offset) << "\n"
-      buf << cscope_mark(record) << record[:key] << "\n"
+      buf << cscope_mark(record) << strip_unicode(record[:key]) << "\n"
 
       buf << CSCOPE_GLOBAL_HACK_START if record[:type] == :func && record[:tbl] == :end
       buf
     rescue ArgumentError
       # invalid utf-8 byte sequence in the line, oh well
-      line
+      strip_unicode(line)
     end
 
     def valid_index?(line, index, key)
@@ -242,10 +240,10 @@ END
       ret.lstrip! if start == 0
       ret.rstrip! if stop == line.length
       ret.gsub!(/\s+/, ' ')
-      ret.empty? ? ' ' : ret
+      ret.empty? ? ' ' : strip_unicode(ret)
     rescue ArgumentError
       # invalid utf-8 byte sequence in the line, oh well
-      line
+      strip_unicode(line)
     end
 
     def cscope_mark(rec)
@@ -260,16 +258,16 @@ END
       when :file
         ret = '@'
       when :defs
-        case rec[:type]
-        when :func
-          ret = '$'
-        when :class, :module
-          ret = 'c'
-        when :type
-          ret = 't'
-        else
-          ret = 'g'
-        end
+        ret = case rec[:type]
+              when :func
+                '$'
+              when :class, :module
+                'c'
+              when :type
+                't'
+              else
+                'g'
+              end
       when :calls
         ret = '`'
       when :requires
@@ -282,7 +280,11 @@ END
         return ''
       end
 
-      "\t" + ret
+      "\t#{ret}"
+    end
+
+    def strip_unicode(str)
+      str.encode(ASCII, invalid: :replace, undef: :replace, replace: '')
     end
   end
 end

data/lib/starscope/fragment_extractor.rb

@@ -5,16 +5,15 @@ module Starscope
       @frags = frags
     end
 
-    def extract(path, text)
-      text = @frags.map { |f| f.delete(:frag).strip }.join("\n")
+    def extract(path, _text)
+      compiled_text = @frags.map { |f| f.delete(:frag).strip }.join("\n")
 
-      extractor_metadata = @child.extract(path, text) do |tbl, name, args|
+      @child.extract(path, compiled_text) do |tbl, name, args|
         args.merge!(@frags[args[:line_no] - 1]) if args[:line_no]
         yield tbl, name, args
       end
 
       # TODO: translate metadata?
-      extractor_metadata
     end
 
     def name