starscope 0.1.10 → 1.0.0

data/.travis.yml

@@ -2,7 +2,6 @@ language: ruby
 
 rvm:
     - 1.8.7
-    - 1.9.2
     - 1.9.3
     - 2.0.0
-    - 2.1.0
+    - 2.1.1

data/CHANGELOG.md

@@ -1,8 +1,33 @@
 Changelog
 =========
 
-v0.1.10 (trunk)
--------------------
+v1.0.0 (trunk)
+--------------------
+
+New Features:
+ * Preliminary export of a few advanced ctags annotations
+ * New -x flag to exclude files from scan (such as compiled .o files)
+ * New --verbose flag for additional output
+
+Bug Fixes:
+ * Correctly write out migrated databases
+ * Be compatible with ruby 1.8 everywhere
+ * Fix golang parsing untyped "var" declarations
+ * Fix golang parsing multi-line literals
+ * Record assignments to ruby constants as definitions
+ * Fix exporting to cscope when scanned files contain invalid unicode
+
+Improvements:
+ * Faster file-type matching
+ * Reworked option flags:
+   * Merged -r and -w into -f
+   * Split -n into --no-read, --no-write, --no-update
+ * New, more flexible database format
+ * Substantially improved searching/matching logic
+ * Miscellanious others via updated dependencies
+
+v0.1.10 (2014-02-24)
+--------------------
 
 Improvements:
  * Import new ruby parser version and make necessary changes so that StarScope

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    starscope (0.1.10)
-      oj (~> 2.5)
+    starscope (1.0.0)
+      oj (~> 2.7)
       parser (~> 2.1)
       ruby-progressbar (~> 1.4)
 
@@ -12,18 +12,18 @@ GEM
     ast (1.1.0)
     coderay (1.1.0)
     method_source (0.8.2)
-    minitest (5.2.3)
-    oj (2.5.5)
-    parser (2.1.5)
+    minitest (5.3.3)
+    oj (2.7.3)
+    parser (2.1.7)
       ast (~> 1.1)
       slop (~> 3.4, >= 3.4.5)
     pry (0.9.12.6)
       coderay (~> 1.0)
       method_source (~> 0.8)
       slop (~> 3.4)
-    rake (10.1.1)
-    ruby-progressbar (1.4.1)
-    slop (3.4.7)
+    rake (10.2.2)
+    ruby-progressbar (1.4.2)
+    slop (3.5.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -10,31 +10,28 @@ only works for C (and sort of works for C++).
 
 StarScope is a similar tool for [Ruby](https://www.ruby-lang.org/) and
 [Golang](http://golang.org/), with a design intended to make it easy to add
-support for other languages at some point within the same framework (thus the
-name StarScope, ie \*scope).
+support for other languages within the same framework (thus the name StarScope,
+i.e. \*scope).
 
 Install it as a gem:
 ```
 $ gem install starscope
 ```
 
-Build your database, by just running it in the project directory:
+Build your database by just running it in the project directory:
 ```
 $ cd ~/my-project
 $ starscope
 ```
 
-Ask it things with `-q`
+Ask it things directly:
 ```
 $ starscope -q calls,new # Lists all callers of new
 ```
 
-Export it for use with your editor
+Export it to various formats for use with your editor:
 ```
 $ starscope -e ctags
-```
-or
-```
 $ starscope -e cscope
 ```
 

data/bin/starscope

@@ -7,18 +7,30 @@ require 'optparse'
 require 'readline'
 require 'starscope'
 
-options = {auto: true, progress: true}
-DEFAULT_DB=".starscope.db"
+DEFAULT_DB='.starscope.db'
+DEFAULT_CTAGS='tags'
+DEFAULT_CSCOPE='cscope.out'
+
+options = {:show_progress => true,
+           :read => true,
+           :write => true,
+           :update => true,
+           :verbose => false,
+           :db => DEFAULT_DB
+}
 
 # Options Parsing
 OptionParser.new do |opts|
   opts.banner = <<END
 Usage: starscope [options] [PATHS]
 
-If you don't pass any of -n, -r, -w or PATHS, default behaviour is to recurse
-in the current directory and build or update the database `#{DEFAULT_DB}`.
+The default database is `#{DEFAULT_DB}` if you don't specify one with -f.
+The default behaviour is to read and update the database.
+If no database exists and no PATHS are specified, StarScope builds a new
+database by recursing in the current directory.
 
-Query scopes must be specified with `::`, for example -q calls,File::mtime.
+Scoped queries must use `::` as the scope separator, even for languages which
+have their own scope syntax.
 END
 
   opts.separator "\nQueries"
@@ -39,14 +51,21 @@ END
   opts.on("-e", "--export FORMAT[,PATH]", "Export in FORMAT to PATH, (see EXPORTING)") do |export|
     options[:export] = export
   end
-  opts.on("-n", "--no-auto", "Don't automatically update/create the DB") do
-    options[:auto] = false
+  opts.on("-f", "--file FILE", "Use FILE instead of `#{DEFAULT_DB}`") do |path|
+    options[:db] = path
   end
-  opts.on("-r", "--read-db PATH", "Reads the DB from PATH instead of default") do |path|
-    options[:read] = path
+  opts.on("-x", "--exclude PATTERN", "Skip files matching PATTERN") do |pattern|
+    options[:exclude] ||= []
+    options[:exclude] << pattern
   end
-  opts.on("-w", "--write-db PATH", "Writes the DB to PATH instead of default") do |path|
-    options[:write] = path
+  opts.on("--no-read", "Don't read the DB from a file") do
+    options[:read] = false
+  end
+  opts.on("--no-write", "Don't write the DB to a file") do
+    options[:write] = false
+  end
+  opts.on("--no-update", "Don't update the DB") do
+    options[:update] = false
   end
 
   opts.separator "\nMisc"
@@ -54,22 +73,25 @@ END
     puts StarScope::VERSION
     exit
   end
+  opts.on("--verbose", "Print extra details") do
+    options[:verbose] = true
+  end
   opts.on("--no-progress", "Don't show progress-bar when updating DB") do
-    options[:progress] = false
+    options[:show_progress] = false
   end
 
   opts.separator <<END
 \nEXPORTING
     At the moment two export formats are supported: 'ctags' and 'cscope'. If
-    you don't specify a path, the output is written to the files 'tags' (for
-    ctags) or 'cscope.out' (for cscope) in the current directory.
+    you don't specify a path, the output is written to the files '#{DEFAULT_CTAGS}' (for
+    ctags) or '#{DEFAULT_CSCOPE}' (for cscope) in the current directory.
 END
 
 end.parse!
 
 def print_summary(db)
   db.summary.each do |name, count|
-    printf("%-8s %5d keys\n", name, count)
+    printf("%-8s %5d records\n", name, count)
   end
 end
 
@@ -78,9 +100,11 @@ def run_query(db, table, value)
     $stderr.puts "Invalid input - no query found."
     return false
   end
-  key, results = db.query(table.to_sym, value)
+  results = db.query(table.to_sym, value)
   if results
-    puts results.map {|val| StarScope::Datum.to_s(val)}
+    results.sort_by {|x| x[:name].join(' ')}.each do |rec|
+      puts StarScope::Record.format(rec)
+    end
   else
     puts "No results found."
   end
@@ -91,10 +115,12 @@ rescue StarScope::DB::NoTableError
 end
 
 def dump(db, table)
-  if table
-    db.dump_table(table.to_sym)
-  else
+  if table.nil?
     db.dump_all
+  elsif table.start_with?('_')
+    db.dump_meta(table[1..-1].to_sym)
+  else
+    db.dump_table(table.to_sym)
   end
   return true
 rescue StarScope::DB::NoTableError
@@ -102,42 +128,49 @@ rescue StarScope::DB::NoTableError
   return false
 end
 
-if options[:auto] and not options[:write]
-  options[:write] = DEFAULT_DB
-  options[:auto_write] = true
-end
+db = StarScope::DB.new(options[:show_progress], options[:verbose])
 
-if File.exists?(DEFAULT_DB) and not options[:read]
-  options[:read] = DEFAULT_DB
-end
+db_exists = File.exists?(options[:db])
 
-db = StarScope::DB.new(options[:progress])
+if options[:read] and db_exists
+  new_data = db.load(options[:db])
+else
+  # no need to run an update if we didn't read any old data
+  options[:update] = false
+end
 
-if options[:read]
-  db.load(options[:read])
-  if !ARGV.empty?
-    db.add_paths(ARGV)
-    new_data = true
-  end
-elsif ARGV.empty?
-  db.add_paths(['.'])
+if options[:exclude]
+  db.add_excludes(options[:exclude])
   new_data = true
-else
+end
+
+if not ARGV.empty?
+  # paths specified, add them
   db.add_paths(ARGV)
   new_data = true
+elsif not (options[:read] and db_exists)
+  # no paths were specified and the database was not read or did not exist;
+  # default to building a new DB in the current directory
+  db.add_paths(['.'])
+  new_data = true
 end
 
-changed = db.update if options[:read] and options[:auto]
+updated = db.update() if options[:update]
+new_data ||= updated
 
-db.save(options[:write]) if options[:write] and (new_data or changed or not options[:auto_write])
+db.save(options[:db]) if options[:write] and (new_data or not db_exists)
 
 if options[:export]
   format, path = options[:export].split(',', 2)
   case format
   when 'ctags'
-    db.export_ctags(path || 'tags')
+    File.open(path || DEFAULT_CTAGS, 'w') do |file|
+      db.export_ctags(file)
+    end
   when 'cscope'
-    db.export_cscope(path || 'cscope.out')
+    File.open(path || DEFAULT_CSCOPE, 'w') do |file|
+      db.export_cscope(file)
+    end
   else
     puts "Unrecognized export format"
   end
@@ -174,29 +207,32 @@ end
 
 if options[:linemode]
   puts "Run your query as 'TABLE QUERY' or run '!help' for more information."
-  while input = Readline.readline("> ", true)
-    cmd, param = input.split(' ', 2)
-    if cmd[0] == '!'
-      case cmd[1..-1]
-      when "dump"
-        dump(db, param)
-      when "summary"
-        print_summary(db)
-      when "update"
-        changed = db.update
-        db.save(options[:write]) if options[:write] and changed
-      when "help"
-        puts linemode_help
-      when "version"
-        puts StarScope::VERSION
-      when "quit"
-        exit
+  begin
+    while input = Readline.readline("> ", true)
+      cmd, param = input.split(' ', 2)
+      if cmd[0] == '!'
+        case cmd[1..-1]
+        when "dump"
+          dump(db, param)
+        when "summary"
+          print_summary(db)
+        when "update"
+          changed = db.update
+          db.save(options[:write]) if options[:write] and changed
+        when "help"
+          puts linemode_help
+        when "version"
+          puts StarScope::VERSION
+        when "quit"
+          exit
+        else
+          puts "Unknown command: '#{input}', try '!help'."
+        end
       else
-        puts "Unknown command: '#{input}', try '!help'."
+        success = run_query(db, cmd, param)
+        puts "Try '!help'." unless success
       end
-    else
-      success = run_query(db, cmd, param)
-      puts "Try '!help'." unless success
     end
+  rescue Interrupt
   end
 end

data/lib/starscope/db.rb

@@ -1,47 +1,56 @@
-require 'starscope/langs/go'
-require 'starscope/langs/ruby'
-require 'starscope/datum'
 require 'date'
 require 'oj'
 require 'zlib'
-require 'ruby-progressbar'
+
+require 'starscope/matcher'
+require 'starscope/output'
+require 'starscope/record'
+
+require 'starscope/langs/coffeescript'
+require 'starscope/langs/go'
+require 'starscope/langs/lua'
+require 'starscope/langs/ruby'
 
 LANGS = [
+  StarScope::Lang::CoffeeScript,
   StarScope::Lang::Go,
+  StarScope::Lang::Lua,
   StarScope::Lang::Ruby
 ]
 
 class StarScope::DB
 
-  DB_FORMAT = 4
-  PBAR_FORMAT = '%t: %c/%C %E ||%b>%i||'
+  DB_FORMAT = 5
 
   class NoTableError < StandardError; end
   class UnknownDBFormatError < StandardError; end
 
-  def initialize(progress)
-    @progress = progress
-    @paths = []
-    @files = {}
+  def initialize(progress, verbose)
+    @output = StarScope::Output.new(progress, verbose)
+    @meta = {:paths => [], :files => {}, :excludes => []}
     @tables = {}
   end
 
+  # returns true if the database had to be up-converted from an old format
   def load(file)
+    @output.log("Reading database from `#{file}`... ")
     File.open(file, 'r') do |file|
       Zlib::GzipReader.wrap(file) do |file|
         format = file.gets.to_i
         if format == DB_FORMAT
-          @paths  = Oj.load(file.gets)
-          @files  = Oj.load(file.gets)
-          @tables = Oj.load(file.gets, :symbol_keys => true)
+          @meta   = Oj.load(file.gets)
+          @tables = Oj.load(file.gets)
+          return false
         elsif format <= 2
           # Old format (pre-json), so read the directories segment then rebuild
           len = file.gets.to_i
           add_paths(Marshal::load(file.read(len)))
-        elsif format < DB_FORMAT
+          return true
+        elsif format <= 4
           # Old format, so read the directories segment then rebuild
           add_paths(Oj.load(file.gets))
-        elsif format > DB_FORMAT
+          return true
+        else
           raise UnknownDBFormatError
         end
       end
@@ -49,57 +58,85 @@ class StarScope::DB
   end
 
   def save(file)
+    @output.log("Writing database to `#{file}`...")
     File.open(file, 'w') do |file|
       Zlib::GzipWriter.wrap(file) do |file|
         file.puts DB_FORMAT
-        file.puts Oj.dump @paths
-        file.puts Oj.dump @files
+        file.puts Oj.dump @meta
         file.puts Oj.dump @tables
       end
     end
   end
 
+  def add_excludes(paths)
+    @meta[:paths] -= paths.map {|p| normalize_glob(p)}
+    paths = paths.map {|p| normalize_fnmatch(p)}
+    @meta[:excludes] += paths
+    @meta[:excludes].uniq!
+    @meta[:files].delete_if do |name, record|
+      if matches_exclude(paths, name)
+        remove_file(name)
+        true
+      else
+        false
+      end
+    end
+  end
+
   def add_paths(paths)
-    paths -= @paths
-    return if paths.empty?
-    @paths += paths
-    files = paths.map {|p| self.class.files_from_path(p)}.flatten
+    @meta[:excludes] -= paths.map {|p| normalize_fnmatch(p)}
+    paths = paths.map {|p| normalize_glob(p)}
+    @meta[:paths] += paths
+    @meta[:paths].uniq!
+    files = Dir.glob(paths).select {|f| File.file? f}
+    files.delete_if {|f| matches_exclude(@meta[:excludes], f)}
     return if files.empty?
-    if @progress
-      pbar = ProgressBar.create(:title => "Building", :total => files.length, :format => PBAR_FORMAT, :length => 80)
-    end
-    files.each do |f|
-      add_file(f)
-      pbar.increment if @progress
-    end
+    @output.new_pbar("Building", files.length)
+    add_new_files(files)
+    @output.finish_pbar
   end
 
   def update
-    new_files = (@paths.map {|p| self.class.files_from_path(p)}.flatten) - @files.keys
-    if @progress
-      pbar = ProgressBar.create(:title => "Updating", :total => new_files.length + @files.length, :format => PBAR_FORMAT, :length => 80)
-    end
-    changed = @files.keys.map do |f|
-      changed = update_file(f)
-      pbar.increment if @progress
-      changed
-    end
-    new_files.each do |f|
-      add_file(f)
-      pbar.increment if @progress
+    new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].keys
+    new_files.delete_if {|f| matches_exclude(@meta[:excludes], f)}
+    @output.new_pbar("Updating", new_files.length + @meta[:files].length)
+    changed = false
+    @meta[:files].delete_if do |name, record|
+      @output.log("Updating `#{name}`")
+      ret = update_file(name)
+      @output.inc_pbar
+      changed = true if ret == :update
+      ret == :delete
     end
-    changed.any? || !new_files.empty?
+    add_new_files(new_files)
+    @output.finish_pbar
+    changed || !new_files.empty?
   end
 
   def dump_table(table)
     raise NoTableError if not @tables[table]
     puts "== Table: #{table} =="
-    @tables[table].sort_by{|k,v| k.downcase}.each do |val, data|
-      puts "#{val}"
-      data.each do |datum|
-        print "\t"
-        puts StarScope::Datum.to_s(datum)
+    @tables[table].sort {|a,b|
+      a[:name][-1].to_s.downcase <=> b[:name][-1].to_s.downcase
+    }.each do |record|
+      puts StarScope::Record.format(record)
+    end
+  end
+
+  def dump_meta(key)
+    if key == :meta
+      puts "== Metadata Summary =="
+      @meta.each do |k, v|
+        puts "#{k}: #{v.count}"
       end
+      return
+    end
+    raise NoTableError if not @meta[key]
+    puts "== Metadata: #{key} =="
+    if @meta[key].is_a? Array
+      @meta[key].sort.each {|x| puts x}
+    else
+      @meta[key].sort.each {|k,v| puts "#{k}: #{v}"}
     end
   end
 
@@ -111,74 +148,56 @@ class StarScope::DB
     ret = {}
 
     @tables.each_key do |key|
-      ret[key] = @tables[key].keys.count
+      ret[key] = @tables[key].count
     end
 
     ret
   end
 
   def query(table, value)
-    fqn = value.split("::")
     raise NoTableError if not @tables[table]
-    key = fqn.last
-    results = @tables[table][key.to_sym] || []
-    if results.empty?
-      matcher = Regexp.new(key, Regexp::IGNORECASE)
-      @tables[table].each do |k,v|
-        if matcher.match(k)
-          results << v
-        end
-      end
-      results.flatten!
-    end
-    return results if results.empty?
-    results.sort! do |a,b|
-      StarScope::Datum.score_match(b, fqn) <=> StarScope::Datum.score_match(a, fqn)
-    end
-    best_score = StarScope::Datum.score_match(results[0], fqn)
-    results = results.select do |result|
-      best_score - StarScope::Datum.score_match(result, fqn) < 4
-    end
-    return fqn.last.to_sym, results
+    input = @tables[table]
+    StarScope::Matcher.new(value, input).query()
   end
 
-  def export_ctags(filename)
-    File.open(filename, 'w') do |file|
-      file.puts <<END
-!_TAG_FILE_FORMAT	2	//
+  def export_ctags(file)
+    file.puts <<END
+!_TAG_FILE_FORMAT	2	/extended format/
 !_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
-!_TAG_PROGRAM_AUTHOR	Evan Huus //
-!_TAG_PROGRAM_NAME	Starscope //
+!_TAG_PROGRAM_AUTHOR	Evan Huus /eapache@gmail.com/
+!_TAG_PROGRAM_NAME	StarScope //
 !_TAG_PROGRAM_URL	https://github.com/eapache/starscope //
 !_TAG_PROGRAM_VERSION	#{StarScope::VERSION}	//
 END
-      defs = (@tables[:defs] || {}).sort
-      defs.each do |key, val|
-        val.each do |entry|
-          file.puts StarScope::Datum.ctag_line(entry)
-        end
-      end
+    defs = (@tables[:defs] || {}).sort_by {|x| x[:name][-1].to_s}
+    defs.each do |record|
+      file.puts StarScope::Record.ctag_line(record)
     end
   end
 
   # ftp://ftp.eeng.dcu.ie/pub/ee454/cygwin/usr/share/doc/mlcscope-14.1.8/html/cscope.html
-  def export_cscope(filename)
+  def export_cscope(file)
     buf = ""
     files = []
-    db_by_line().each do |file, lines|
+    db_by_line().each do |filename, lines|
       if not lines.empty?
-        buf << "\t@#{file}\n\n"
-        files << file
+        buf << "\t@#{filename}\n\n"
+        files << filename
       end
-      lines.sort.each do |line_no, vals|
-        line = vals.first[:entry][:line].strip.gsub(/\s+/, ' ')
+      lines.sort.each do |line_no, records|
+        begin
+          line = records.first[:line].strip.gsub(/\s+/, ' ')
+        rescue ArgumentError
+          # invalid utf-8 byte sequence in the line, just do our best
+          line = records.first[:line]
+        end
         toks = {}
 
-        vals.each do |val|
-          index = line.index(val[:key].to_s)
+        records.each do |record|
+          index = line.index(record[:name][-1].to_s)
           while index
-            toks[index] = val
-            index = line.index(val[:key].to_s, index + 1)
+            toks[index] = record
+            index = line.index(record[:name][-1].to_s, index + 1)
           end
         end
 
@@ -186,11 +205,11 @@ END
 
         prev = 0
         buf << line_no.to_s << " "
-        toks.sort().each do |offset, val|
+        toks.sort().each do |offset, record|
           buf << line.slice(prev...offset) << "\n"
-          buf << StarScope::Datum.cscope_mark(val[:tbl], val[:entry])
-          buf << val[:key].to_s << "\n"
-          prev = offset + val[:key].to_s.length
+          buf << StarScope::Record.cscope_mark(record[:tbl], record)
+          buf << record[:name][-1].to_s << "\n"
+          prev = offset + record[:name][-1].to_s.length
         end
         buf << line.slice(prev..-1) << "\n\n"
       end
@@ -199,86 +218,100 @@ END
     buf << "\t@\n"
 
     header = "cscope 15 #{Dir.pwd} -c "
-    offset = "%010d\n" % (header.length + 11 + buf.length)
-
-    File.open(filename, 'w') do |file|
-      file.print(header)
-      file.print(offset)
-      file.print(buf)
-
-      file.print("#{@paths.length}\n")
-      @paths.each {|p| file.print("#{p}\n")}
-      file.print("0\n")
-      file.print("#{files.length}\n")
-      buf = ""
-      files.each {|f| buf << f + "\n"}
-      file.print("#{buf.length}\n#{buf}")
-    end
+    offset = "%010d\n" % (header.length + 11 + buf.bytes.to_a.length)
+
+    file.print(header)
+    file.print(offset)
+    file.print(buf)
+
+    file.print("#{@meta[:paths].length}\n")
+    @meta[:paths].each {|p| file.print("#{p}\n")}
+    file.print("0\n")
+    file.print("#{files.length}\n")
+    buf = ""
+    files.each {|f| buf << f + "\n"}
+    file.print("#{buf.length}\n#{buf}")
   end
 
   private
 
-  def self.files_from_path(path)
-    if File.file?(path)
-      [path]
+  def add_new_files(files)
+    files.each do |file|
+      @output.log("Adding `#{file}`")
+      @meta[:files][file] = {}
+      parse_file(file)
+      @output.inc_pbar
+    end
+  end
+
+  # File.fnmatch treats a "**" to match files and directories recursively
+  def normalize_fnmatch(path)
+    if path == "."
+      "**"
+    elsif File.directory?(path)
+      File.join(path, "**")
+    else
+      path
+    end
+  end
+
+  # Dir.glob treats a "**" to only match directories recursively; you need
+  # "**/*" to match all files recursively
+  def normalize_glob(path)
+    if path == "."
+      File.join("**", "*")
     elsif File.directory?(path)
-      Dir[File.join(path, "**", "*")].select {|p| File.file?(p)}
+      File.join(path, "**", "*")
     else
-      []
+      path
     end
   end
 
   def db_by_line()
-    tmpdb = {}
-    @tables.each do |tbl, vals|
-      vals.each do |key, val|
-        val.each do |entry|
-          if entry[:line_no]
-            tmpdb[entry[:file]] ||= {}
-            tmpdb[entry[:file]][entry[:line_no]] ||= []
-            tmpdb[entry[:file]][entry[:line_no]] << {:tbl => tbl, :key => key, :entry => entry}
-          end
-        end
+    db = {}
+    @tables.each do |tbl, records|
+      records.each do |record|
+        next if not record[:line_no]
+        record[:tbl] = tbl
+        db[record[:file]] ||= {}
+        db[record[:file]][record[:line_no]] ||= []
+        db[record[:file]][record[:line_no]] << record
       end
     end
-    return tmpdb
+    return db
   end
 
-  def add_file(file)
-    return if not File.file? file
+  def matches_exclude(patterns, file)
+    patterns.map {|p| File.fnmatch(p, file)}.any?
+  end
 
-    @files[file] = File.mtime(file).to_s
+  def parse_file(file)
+    @meta[:files][file][:last_updated] = File.mtime(file).to_i
 
     LANGS.each do |lang|
       next if not lang.match_file file
-      lang.extract file do |tbl, key, args|
-        key = key.to_sym
-        @tables[tbl] ||= {}
-        @tables[tbl][key] ||= []
-        @tables[tbl][key] << StarScope::Datum.build(file, key, args)
+      lang.extract file do |tbl, name, args|
+        @tables[tbl] ||= []
+        @tables[tbl] << StarScope::Record.build(file, name, args)
       end
+      @meta[:files][file][:lang] = lang.name.split('::').last.to_sym
     end
   end
 
   def remove_file(file)
-    @files.delete(file)
     @tables.each do |name, tbl|
-      tbl.each do |key, val|
-        val.delete_if {|dat| dat[:file] == file}
-      end
+      tbl.delete_if {|val| val[:file] == file}
     end
   end
 
   def update_file(file)
     if not File.exists?(file) or not File.file?(file)
       remove_file(file)
-      true
-    elsif DateTime.parse(@files[file]).to_time.to_i < File.mtime(file).to_i
+      :delete
+    elsif @meta[:files][file][:last_updated] < File.mtime(file).to_i
       remove_file(file)
-      add_file(file)
-      true
-    else
-      false
+      parse_file(file)
+      :update
     end
   end