starscope 1.1.2 → 1.2.0

data/lib/starscope/db.rb

@@ -3,103 +3,83 @@ require 'oj'
 require 'set'
 require 'zlib'
 
+require 'starscope/export'
 require 'starscope/matcher'
 require 'starscope/output'
-require 'starscope/record'
-
-# cscope has this funky issue where it refuses to recognize function calls that
-# happen outside of a function definition - this isn't an issue in C, where all
-# calls must occur in a function, but in ruby et al. it is perfectly legal to
-# write normal code outside the "scope" of a function definition - we insert a
-# fake shim "global" function everywhere we can to work around this
-CSCOPE_GLOBAL_HACK_START = "\n\t$-\n"
-CSCOPE_GLOBAL_HACK_STOP = "\n\t}\n"
 
 # dynamically load all our language extractors
-LANGS = []
+LANGS = {}
+EXTRACTORS = []
 Dir.glob("#{File.dirname(__FILE__)}/langs/*.rb").each do |path|
   require path
-  lang = /(\w+)\.rb$/.match(path)[1]
-  LANGS << eval("StarScope::Lang::#{lang.capitalize}")
+  lang = /(\w+)\.rb$/.match(path)[1].capitalize
+  mod_name = "Starscope::Lang::#{lang}"
+  EXTRACTORS << eval(mod_name)
+  LANGS[lang.to_sym] = eval("#{mod_name}::VERSION")
 end
 
-class StarScope::DB
+class Starscope::DB
+
+  include Starscope::Export
 
   DB_FORMAT = 5
 
   class NoTableError < StandardError; end
   class UnknownDBFormatError < StandardError; end
 
-  def initialize(output_level)
-    @output = StarScope::Output.new(output_level)
+  def initialize(output)
+    @output = output
     @meta = {:paths => [], :files => {}, :excludes => [],
-             :version => StarScope::VERSION}
+             :langs => LANGS, :version => Starscope::VERSION}
     @tables = {}
   end
 
-  # returns true if the database had to be up-converted from an old format
-  def load(file)
-    @output.log("Reading database from `#{file}`... ")
-    File.open(file, 'r') do |file|
-      Zlib::GzipReader.wrap(file) do |file|
-        case file.gets.to_i
-        when DB_FORMAT
-          @meta   = Oj.load(file.gets)
-          @tables = Oj.load(file.gets)
-          return false
-        when 3..4
-          # Old format, so read the directories segment then rebuild
-          add_paths(Oj.load(file.gets))
-          return true
-        when 0..2
-          # Old format (pre-json), so read the directories segment then rebuild
-          len = file.gets.to_i
-          add_paths(Marshal::load(file.read(len)))
-          return true
-        else
-          raise UnknownDBFormatError
-        end
-      end
-    end
+  def load(filename)
+    @output.extra("Reading database from `#{filename}`... ")
+    current_fmt = open_db(filename)
+    fixup if current_fmt
+    current_fmt
   end
 
-  def save(file)
-    @output.log("Writing database to `#{file}`...")
+  def save(filename)
+    @output.extra("Writing database to `#{filename}`...")
 
     # regardless of what the old version was, the new version is written by us
-    @meta[:version] = StarScope::VERSION
+    @meta[:version] = Starscope::VERSION
+
+    @meta[:langs].merge!(LANGS)
 
-    File.open(file, 'w') do |file|
-      Zlib::GzipWriter.wrap(file) do |file|
-        file.puts DB_FORMAT
-        file.puts Oj.dump @meta
-        file.puts Oj.dump @tables
+    File.open(filename, 'w') do |file|
+      Zlib::GzipWriter.wrap(file) do |stream|
+        stream.puts DB_FORMAT
+        stream.puts Oj.dump @meta
+        stream.puts Oj.dump @tables
       end
     end
   end
 
   def add_excludes(paths)
-    @output.log("Excluding files in paths #{paths}...")
-    @meta[:paths] -= paths.map {|p| normalize_glob(p)}
-    paths = paths.map {|p| normalize_fnmatch(p)}
+    @output.extra("Excluding files in paths #{paths}...")
+    @meta[:paths] -= paths.map {|p| self.class.normalize_glob(p)}
+    paths = paths.map {|p| self.class.normalize_fnmatch(p)}
     @meta[:excludes] += paths
     @meta[:excludes].uniq!
 
-    excluded = @meta[:files].keys.select {|name| matches_exclude?(paths, name)}
+    excluded = @meta[:files].keys.select {|name| matches_exclude?(name, paths)}
     remove_files(excluded)
   end
 
   def add_paths(paths)
-    @output.log("Adding files in paths #{paths}...")
-    @meta[:excludes] -= paths.map {|p| normalize_fnmatch(p)}
-    paths = paths.map {|p| normalize_glob(p)}
+    @output.extra("Adding files in paths #{paths}...")
+    @meta[:excludes] -= paths.map {|p| self.class.normalize_fnmatch(p)}
+    paths = paths.map {|p| self.class.normalize_glob(p)}
     @meta[:paths] += paths
     @meta[:paths].uniq!
     files = Dir.glob(paths).select {|f| File.file? f}
-    files.delete_if {|f| matches_exclude?(@meta[:excludes], f)}
+    files.delete_if {|f| matches_exclude?(f)}
     return if files.empty?
     @output.new_pbar("Building", files.length)
-    add_new_files(files)
+    add_files(files)
     @output.finish_pbar
   end
 
@@ -109,189 +89,106 @@ class StarScope::DB
     changes[:deleted] ||= []
 
     new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].keys
-    new_files.delete_if {|f| matches_exclude?(@meta[:excludes], f)}
+    new_files.delete_if {|f| matches_exclude?(f)}
 
     if changes[:deleted].empty? && changes[:modified].empty? && new_files.empty?
-      @output.print("No changes detected.")
+      @output.normal("No changes detected.")
       return false
     end
 
     @output.new_pbar("Updating", changes[:modified].length + new_files.length)
     remove_files(changes[:deleted])
     update_files(changes[:modified])
-    add_new_files(new_files)
+    add_files(new_files)
     @output.finish_pbar
 
     true
   end
 
-  def dump_table(table)
+  def query(table, value)
     raise NoTableError if not @tables[table]
-
-    puts "== Table: #{table} =="
-    puts "No records" if @tables[table].empty?
-
-    @tables[table].sort {|a,b|
-      a[:name][-1].to_s.downcase <=> b[:name][-1].to_s.downcase
-    }.each do |record|
-      puts StarScope::Record.format(record)
-    end
-  end
-
-  def dump_meta(key)
-    if key == :meta
-      puts "== Metadata Summary =="
-      @meta.each do |k, v|
-        print "#{k}: "
-        if [Array, Hash].include? v.class
-          puts v.count
-        else
-          puts v
-        end
-      end
-      return
-    end
-    raise NoTableError if not @meta[key]
-    puts "== Metadata: #{key} =="
-    if @meta[key].is_a? Array
-      @meta[key].sort.each {|x| puts x}
-    elsif @meta[key].is_a? Hash
-      @meta[key].sort.each {|k,v| puts "#{k}: #{v}"}
-    else
-      puts @meta[key]
-    end
+    input = @tables[table]
+    Starscope::Matcher.new(value, input).query()
   end
 
-  def dump_all
-    @tables.keys.each {|tbl| dump_table(tbl)}
-  end
+  def line_for_record(rec)
+    return rec[:line] if rec[:line]
 
-  def summary
-    ret = {}
+    file = @meta[:files][rec[:file]]
 
-    @tables.each_key do |key|
-      ret[key] = @tables[key].count
-    end
+    return file[:lines][rec[:line_no]-1] if file[:lines]
+  end
 
-    ret
+  def tables
+    @tables.keys
   end
 
-  def query(table, value)
+  def records(table)
     raise NoTableError if not @tables[table]
-    input = @tables[table]
-    StarScope::Matcher.new(value, input).query()
-  end
 
-  def export_ctags(file)
-    file.puts <<END
-!_TAG_FILE_FORMAT	2	/extended format/
-!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
-!_TAG_PROGRAM_AUTHOR	Evan Huus /eapache@gmail.com/
-!_TAG_PROGRAM_NAME	StarScope //
-!_TAG_PROGRAM_URL	https://github.com/eapache/starscope //
-!_TAG_PROGRAM_VERSION	#{StarScope::VERSION}	//
-END
-    defs = (@tables[:defs] || {}).sort_by {|x| x[:name][-1].to_s}
-    defs.each do |record|
-      file.puts StarScope::Record.ctag_line(record, @meta[:files][record[:file]])
-    end
+    @tables[table]
   end
 
-  # ftp://ftp.eeng.dcu.ie/pub/ee454/cygwin/usr/share/doc/mlcscope-14.1.8/html/cscope.html
-  def export_cscope(file)
-    buf = ""
-    files = []
-    db_by_line().each do |filename, lines|
-      next if lines.empty?
-
-      buf << "\t@#{filename}\n\n"
-      buf << "0 #{CSCOPE_GLOBAL_HACK_START}\n"
-      files << filename
-      func_count = 0
-
-      lines.sort.each do |line_no, records|
-        line = records.first[:line]
-        toks = tokenize_line(line, records)
-        next if toks.empty?
-
-        prev = 0
-        buf << line_no.to_s << " "
-        toks.each do |offset, record|
-
-          next if offset < prev # this probably indicates an extractor bug
-
-          # Don't export nested functions, cscope barfs on them since C doesn't
-          # have them at all. Skipping tokens is easy; since prev isn't updated
-          # they get turned into plain text automatically.
-          if record[:type] == :func
-            case record[:tbl]
-            when :defs
-              func_count += 1
-              next unless func_count == 1
-            when :end
-              func_count -= 1
-              next unless func_count == 0
-            end
-          end
-
-          buf << CSCOPE_GLOBAL_HACK_STOP if record[:type] == :func && record[:tbl] == :defs
-          buf << cscope_plaintext(line, prev, offset) << "\n"
-          buf << StarScope::Record.cscope_mark(record[:tbl], record) << record[:key] << "\n"
-          buf << CSCOPE_GLOBAL_HACK_START if record[:type] == :func && record[:tbl] == :end
-
-          prev = offset + record[:key].length
+  def metadata(key=nil)
+    return @meta.keys if key.nil?
 
-        end
-        buf << cscope_plaintext(line, prev, line.length) << "\n\n"
-      end
-    end
-
-    buf << "\t@\n"
-
-    header = "cscope 15 #{Dir.pwd} -c "
-    offset = "%010d\n" % (header.length + 11 + buf.bytes.count)
+    raise NoTableError unless @meta[key]
 
-    file.print(header)
-    file.print(offset)
-    file.print(buf)
-
-    file.print("#{@meta[:paths].length}\n")
-    @meta[:paths].each {|p| file.print("#{p}\n")}
-    file.print("0\n")
-    file.print("#{files.length}\n")
-    buf = ""
-    files.each {|f| buf << f + "\n"}
-    file.print("#{buf.length}\n#{buf}")
+    @meta[key]
   end
 
   private
 
-  def add_new_files(files)
-    files.each do |file|
-      @output.log("Adding `#{file}`")
-      parse_file(file)
-      @output.inc_pbar
+  def open_db(filename)
+    File.open(filename, 'r') do |file|
+      begin
+        Zlib::GzipReader.wrap(file) do |stream|
+          parse_db(stream)
+        end
+      rescue Zlib::GzipFile::Error
+        file.rewind
+        parse_db(file)
+      end
     end
   end
 
-  def update_files(files)
-    remove_files(files)
-    add_new_files(files)
+  # returns true iff the database is in the most recent format
+  def parse_db(stream)
+    case stream.gets.to_i
+    when DB_FORMAT
+      @meta   = Oj.load(stream.gets)
+      @tables = Oj.load(stream.gets)
+      return true
+    when 3..4
+      # Old format, so read the directories segment then rebuild
+      add_paths(Oj.load(stream.gets))
+      return false
+    when 0..2
+      # Old format (pre-json), so read the directories segment then rebuild
+      len = stream.gets.to_i
+      add_paths(Marshal::load(stream.read(len)))
+      return false
+    else
+      raise UnknownDBFormatError
+    end
+  rescue Oj::ParseError
+    stream.rewind
+    raise unless stream.gets.to_i == DB_FORMAT
+    # try reading as formated json, which is much slower, but it is sometimes
+    # useful to be able to directly read your db
+    objects = []
+    Oj.load(stream) {|obj| objects << obj}
+    @meta, @tables = objects
+    return true
   end
 
-  def remove_files(files)
-    files.each do |file|
-      @output.log("Removing `#{file}`")
-      @meta[:files].delete(file)
-    end
-    files = files.to_set
-    @tables.each do |name, tbl|
-      tbl.delete_if {|val| files.include?(val[:file])}
-    end
+  def fixup
+    # misc things that were't worth bumping the format for, but which might not be written by old versions
+    @meta[:langs] ||= {}
   end
 
   # File.fnmatch treats a "**" to match files and directories recursively
-  def normalize_fnmatch(path)
+  def self.normalize_fnmatch(path)
     if path == "."
       "**"
     elsif File.directory?(path)
@@ -303,7 +200,7 @@ END
 
   # Dir.glob treats a "**" to only match directories recursively; you need
   # "**/*" to match all files recursively
-  def normalize_glob(path)
+  def self.normalize_glob(path)
     if path == "."
       File.join("**", "*")
     elsif File.directory?(path)
@@ -313,93 +210,81 @@ END
     end
   end
 
-  def db_by_line()
-    db = {}
-    @tables.each do |tbl, records|
-      records.each do |record|
-        next if not record[:line_no]
-        record[:tbl] = tbl
-        db[record[:file]] ||= {}
-        db[record[:file]][record[:line_no]] ||= []
-        db[record[:file]][record[:line_no]] << record
-      end
-    end
-    return db
+  def matches_exclude?(file, patterns = @meta[:excludes])
+    patterns.map {|p| File.fnmatch(p, file)}.any?
   end
 
-  def tokenize_line(line, records)
-    toks = {}
-
-    records.each do |record|
-      key = record[:name][-1].to_s
-
-      # use the column if we have it, otherwise fall back to scanning
-      index = record[:col] || line.index(key)
-
-      # keep scanning if our current index doesn't actually match the key, or if
-      # either the preceeding or succeeding character is a word character
-      # (meaning we've accidentally matched the middle of some other token)
-      while !index.nil? &&
-        ((line[index, key.length] != key) ||
-         (index > 0 && line[index-1] =~ /\w/) ||
-         (index+key.length < line.length && line[index+key.length] =~ /\w/))
-        index = line.index(key, index+1)
-      end
-
-      next if index.nil?
-
-      # Strip trailing non-word characters, otherwise cscope barfs on
-      # function names like `include?`
-      if key =~ /^\W*$/
-        next unless [:defs, :end].include?(record[:tbl])
-      else
-        key.sub!(/\W+$/, '')
-      end
-
-      record[:key] = key
-      toks[index] = record
-
+  def add_files(files)
+    files.each do |file|
+      @output.extra("Adding `#{file}`")
+      parse_file(file)
+      @output.inc_pbar
     end
-
-    return toks.sort
   end
 
-  def cscope_plaintext(line, start, stop)
-    ret = line.slice(start, stop-start)
-    ret.lstrip! if start == 0
-    ret.rstrip! if stop == line.length
-    ret.gsub(/\s+/, ' ')
-  rescue ArgumentError
-    # invalid utf-8 byte sequence in the line, oh well
-    line
+  def remove_files(files)
+    files.each do |file|
+      @output.extra("Removing `#{file}`")
+      @meta[:files].delete(file)
+    end
+    files = files.to_set
+    @tables.each do |name, tbl|
+      tbl.delete_if {|val| files.include?(val[:file])}
+    end
   end
 
-  def matches_exclude?(patterns, file)
-    patterns.map {|p| File.fnmatch(p, file)}.any?
+  def update_files(files)
+    remove_files(files)
+    add_files(files)
   end
 
   def parse_file(file)
     @meta[:files][file] = {:last_updated => File.mtime(file).to_i}
 
-    LANGS.each do |lang|
-      next if not lang.match_file file
-      lang.extract file do |tbl, name, args|
+    EXTRACTORS.each do |extractor|
+      next if not extractor.match_file file
+
+      lines = nil
+      line_cache = nil
+      extractor.extract file do |tbl, name, args|
         @tables[tbl] ||= []
-        @tables[tbl] << StarScope::Record.build(file, name, args)
+        @tables[tbl] << self.class.normalize_record(file, name, args)
+
+        if args[:line_no]
+          line_cache ||= File.readlines(file)
+          lines ||= Array.new(line_cache.length)
+          lines[args[:line_no]-1] = line_cache[args[:line_no]-1].chomp
+        end
       end
-      @meta[:files][file][:lang] = lang.name.split('::').last.to_sym
+
+      @meta[:files][file][:lang] = extractor.name.split('::').last.to_sym
+      @meta[:files][file][:lines] = lines
       return
     end
   end
 
   def file_changed(name)
-    if not File.exists?(name) or not File.file?(name)
+    file_meta = @meta[:files][name]
+    if !File.exists?(name) || !File.file?(name)
       :deleted
-    elsif @meta[:files][name][:last_updated] < File.mtime(name).to_i
+    elsif (file_meta[:last_updated] < File.mtime(name).to_i) ||
+          (file_meta[:lang] && (@meta[:langs][file_meta[:lang]] || 0) < LANGS[file_meta[:lang]])
       :modified
     else
       :unchanged
     end
   end
 
+  def self.normalize_record(file, name, args)
+    args[:file] = file
+
+    if name.is_a? Array
+      args[:name] = name.map {|x| x.to_sym}
+    else
+      args[:name] = [name.to_sym]
+    end
+
+    args
+  end
+
 end