fastri 0.1.1.1 → 0.2.0.1

data/CHANGES

@@ -1,6 +1,18 @@
 
 User-visible changes in FastRI
 
+Since version 0.1.1 (2006-11-10)
+================================
+Features
+--------
+* fri can do full-text search (-S, --full-text); try  fri -S byte order
+* fri can now determine where a method actually came from for core classes
+  e.g.  fri File.inject   -> docs for Enumerable#inject
+* you can specify which ports the DRb services must bind to:
+    fastri-server -s 192.168.1.2:54321
+    fri -s 192.168.1.2:12345
+* new search methods: "anywhere" (a) and "anywhere, case-indep." (A)
+
 Since version 0.1.0 (2006-11-08)
 ================================
 Features

data/Rakefile

@@ -42,10 +42,11 @@ Spec = Gem::Specification.new do |s|
   s.summary = "RI docs across machines, faster and smarter than ri."
   s.description = <<EOF
 FastRI is an alternative to the ri command-line tool. It is *much* faster, and
-also allows you to offer RI lookup services over DRb. FastRI is a bit smarter
-than ri, and can find classes anywhere in the hierarchy without specifying the
-"full path". It also knows about gems, and can tell you e.g. which extensions
-to a core class were added by a specific gem.
+also allows you to offer RI lookup services over DRb. FastRI is smarter than
+ri, and can find classes anywhere in the hierarchy without specifying the
+"full path". FastRI can perform fast full-text searches. It also knows about
+gems, and can tell you e.g. which extensions to a core class were added by a
+specific gem.
 EOF
   s.files = PKG_FILES.to_a
   s.require_path = 'lib'

data/bin/fastri-server

@@ -1,10 +1,12 @@
-#! /home/batsman/usr//bin/ruby
+#!/usr/bin/env ruby
 # fastri-server: serve RI documentation over DRb 
 # Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
 
 require 'fastri/version'
 require 'fastri/ri_index'
 require 'fastri/ri_service'
+require 'fastri/util'
+require 'fastri/full_text_indexer'
 require 'enumerator'
 
 FASTRI_SERVER_VERSION = "0.0.1"
@@ -18,25 +20,10 @@ def make_index(index_file)
   paths = [ RI::Paths::SYSDIR, RI::Paths::SITEDIR, RI::Paths::HOMEDIR ].find_all do |p|
     p && File.directory?(p)
   end
-  begin
-    require 'rubygems'
-    gemdirs = Dir["#{Gem.path}/doc/*/ri"]
-    gems = Hash.new{|h,k| h[k] = []}
-    gemdirs.each do |path|
-      gemname, version = %r{/([^/]+)-(.*)/ri$}.match(path).captures
-      if gemname.nil? # doesn't follow any conventions :(
-        gems[path[%r{/([^/]+)/ri$}, 1]] << ["unknown", path]
-      else
-        gems[gemname] << [version, path]
-      end
-    end
-    gems.sort_by{|name, _| name}.each do |name, versions|
-      version, path = versions.sort.last
-      puts "Indexing RI docs for #{name} version #{version}."
-      paths << path
-    end
-  rescue LoadError
-  end 
+  FastRI::Util.gem_directories_unique.each do |name, version, path|
+    paths << path
+    puts "Indexing RI docs for #{name} version #{version || "unknown"}."
+  end
 
   puts "Building index."
   t0 = Time.new
@@ -52,21 +39,59 @@ EOF
   ri_reader
 end
 
-# stolen from RubyGems
-def find_home
-  ['HOME', 'USERPROFILE'].each do |homekey|
-    return ENV[homekey] if ENV[homekey]
+def linearize(comment)
+  case s = comment["body"]
+  when String; s
+  else 
+    if Array === (y = comment["contents"])
+      y.map{|z| linearize(z)}.join("\n")
+    elsif s = comment["text"]
+      s
+    else
+      nil
+    end
+  end
+end
+
+def make_full_text_index(dir)
+  paths = [ RI::Paths::SYSDIR, RI::Paths::SITEDIR, RI::Paths::HOMEDIR ].find_all do |p|
+    p && File.directory?(p)
   end
-  if ENV['HOMEDRIVE'] && ENV['HOMEPATH']
-    return "#{ENV['HOMEDRIVE']}:#{ENV['HOMEPATH']}"
+  FastRI::Util.gem_directories_unique.each do |name, version, path|
+    paths << path
+    puts "Indexing RI docs for #{name} version #{version || "unknown"}."
   end
-  begin
-    File.expand_path("~")
-  rescue StandardError => ex
-    if File::ALT_SEPARATOR
-      "C:/"
-    else
-      "/"
+  unless File.exist?(dir)
+    Dir.mkdir(dir)
+  end
+  indexer = FastRI::FullTextIndexer.new(40)
+  bad = 0
+  paths.each do |path|
+    Dir["#{path}/**/*.yaml"].each do |yamlfile|
+      yaml = File.read(yamlfile)
+      begin
+        data = YAML.load(yaml.gsub(/ \!.*/, ''))
+      rescue Exception
+        bad += 1
+        #puts "Couldn't load #{yamlfile}"
+        next
+      end
+
+      desc = (data['comment']||[]).map{|x| linearize(x)}.join("\n")
+      desc.gsub!(/<\/?(em|b|tt|ul|ol|table)>/, "")
+      desc.gsub!(/&quot;/, "'")
+      desc.gsub!(/&lt;/, "<")
+      desc.gsub!(/&gt;/, ">")
+      desc.gsub!(/&amp;/, "&")
+      unless desc.empty?
+        indexer.add_document(yamlfile, desc) 
+      end
+    end
+  end
+
+  File.open(File.join(dir, "full_text.dat"), "wb") do |fulltextIO|
+    File.open(File.join(dir, "suffixes.dat"), "wb") do |suffixesIO|
+      indexer.build_index(fulltextIO, suffixesIO)
     end
   end
 end
@@ -75,8 +100,12 @@ end
 
 require 'optparse'
 
+home = FastRI::Util.find_home
 options = {:allowed_hosts => ["127.0.0.1"], :addr => "127.0.0.1",
-  :index_file => File.join(find_home, ".fastri-index")}
+  :index_file => File.join(home, ".fastri-index"),
+  :do_full_text => false, 
+  :full_text_dir => File.join(home, ".fastri-fulltext"),
+}
 OptionParser.new do |opts|
   opts.banner = "Usage: fastri-server.rb [options]"
 
@@ -100,6 +129,17 @@ OptionParser.new do |opts|
     exit 0
   end
 
+  opts.on("-F", "--full-text-dir DIR", "Place full-text index in DIR",
+          "(default: #{options[:full_text_dir]})") do |dir|
+    options[:full_text_dir] = dir if dir
+    options[:do_full_text]  = true
+  end
+
+  opts.on("-B", "--rebuild-full-text", "Rebuild full-text index.") do
+    make_full_text_index(options[:full_text_dir])
+    exit 0
+  end
+
   opts.on("-h", "--help", "Show this help message") do 
     puts opts
     exit
@@ -129,7 +169,9 @@ options[:allowed_hosts].each{|host| acl_opt.concat ["allow", host.strip]}
 acl = ACL.new(acl_opt)
 DRb.install_acl(acl)
 
-drb_addr = "druby://#{options[:addr]}:0"
+ip   = options[:addr][/^[^:]+/]    || "127.0.0.1"
+port = options[:addr][/:(\d+)/, 1] || 0
+drb_addr = "druby://#{ip}:#{port}"
 DRb.start_service(drb_addr)
 
 $SAFE = 1

data/bin/fri

@@ -1,10 +1,12 @@
-#! /home/batsman/usr//bin/ruby
+#!/usr/bin/env ruby
 # fri: access RI documentation through DRb
 # Copyright (C) 2006 Mauricio Fernandez <mfp@acm.org>
 #
 
 require 'rinda/ring'
 require 'optparse'
+require 'fastri/util'
+require 'fastri/full_text_index'
 
 # we bind to 127.0.0.1 by default, because otherwise Ruby will try with
 # 0.0.0.0, which results in a DNS request, adding way too much latency
@@ -27,6 +29,8 @@ options = {
     :nested_partial, :nested_partial_ci,
   ],
   :show_matches => false,
+  :do_full_text => false, 
+  :full_text_dir => File.join(FastRI::Util.find_home, ".fastri-fulltext"),
 }
 override_addr_env = false
 optparser = OptionParser.new do |opts|
@@ -41,17 +45,28 @@ optparser = OptionParser.new do |opts|
   order_mapping = {
     'e' => :exact,   'E' => :exact_ci,   'n' => :nested, 'N' => :nested_ci,
     'p' => :partial, 'P' => :partial_ci, 'x' => :nested_partial,
-    'X' => :nested_partial_ci
+    'X' => :nested_partial_ci, 'a' => :anywhere, 'A' => :anywhere_ci,
   }
   opts.on("-O", "--order ORDER", "Specify lookup order.",
           "(default: eEnNpPxX)", "Uppercase: case-indep.",
-          "e: exact, n: nested, p: partial",
-          "x: nested and partial") do |order|
+          "e:exact n:nested p:partial (completion)",
+          "x:nested and partial",
+          "a:match method name anywhere") do |order|
     options[:lookup_order] = order.split(//).map{|x| order_mapping[x]}.compact
   end
 
   opts.on("--show-matches", "Only show matching entries."){ options[:show_matches] = true }
 
+  opts.on("-S", "--full-text", "Perform full-text search.") do 
+    options[:do_full_text] = true
+  end
+
+  opts.on("-F", "--full-text-dir DIR", "Use full-text index in DIR",
+          "(default: #{options[:full_text_dir]})") do |dir|
+    options[:full_text_dir] = dir if dir
+    options[:do_full_text]  = true
+  end
+
   opts.on("-f", "--format FMT", "Format to use when displaying output:",
           "   ansi, plain (default: #{options[:format]})") do |format|
     options[:format] = format
@@ -74,12 +89,53 @@ if ARGV.empty?
   exit
 end
 
+# {{{ try to find where the method comes from exactly
+def help_method_extract(m) # :nodoc:
+  unless m.inspect =~ %r[\A#<(?:Unbound)?Method: (.*?)>\Z]
+    raise "Cannot parse result of #{m.class}#inspect: #{m.inspect}"
+  end
+  $1.sub(/\A.*?\((.*?)\)(.*)\Z/){ "#{$1}#{$2}" }.sub(/\./, "::").sub(/#<Class:(.*?)>#/) { "#{$1}::" }
+end
+
+def magic_help(query)
+  if query =~ /\A(.*?)(#|::|\.)(.*)\Z/
+    c, k, m = $1, $2, $3
+    begin
+      c = Object.const_get(c)
+      m = case k
+      when "#"
+        c.instance_method(m)
+      when "::"
+        c.method(m)
+      when "."
+        begin
+          c.method(m)
+        rescue NameError
+          c.instance_method(m)
+        end
+      end
+      help_method_extract(m)
+    rescue Exception
+      query
+    end
+  else
+    query
+  end
+end
+help_query = magic_help(ARGV[0])
+
+#{{{ determine the address to bind to
 if override_addr_env
-  addr = "druby://#{options[:addr]}:0"
+  addr_spec = options[:addr]
 else
-  addr =  "druby://#{ENV["FASTRI_ADDR"]||options[:addr]}:0"
+  addr_spec = ENV["FASTRI_ADDR"] || options[:addr]
 end
 
+ip   = addr_spec[/^[^:]+/]    || "127.0.0.1"
+port = addr_spec[/:(\d+)/, 1] || 0
+addr = "druby://#{ip}:#{port}"
+
+#{{{ start DRb and perform request
 begin
   DRb.start_service(addr)
   ring_server = Rinda::RingFinger.primary
@@ -103,9 +159,90 @@ info_options = {
   :width     => options[:width],
   :lookup_order => options[:lookup_order],
 }
+
+MAX_CONTEXT_LINES = 20
+def context_wrap(text, width)
+  "... " + 
+  text.gsub(/(.{1,#{width-4}})( +|$\n?)|(.{1,#{width-4}})/, "\\1\\3\n").chomp
+end
+
+def display_fulltext_search_results(results, gem_dir_info = FastRI::Util.gem_directories_unique,
+                                    width = 78)
+  return if results.empty?
+  path = File.expand_path(results[0].path)
+  gem_name, version, gem_path = FastRI::Util.gem_info_for_path(path, gem_dir_info)
+  if gem_name
+    rel_path = path[/#{Regexp.escape(gem_path)}\/(.*)/, 1]
+    if rel_path
+      entry_name = FastRI::Util.gem_relpath_to_full_name(rel_path)
+    end
+    puts "Found in #{gem_name} #{version}  #{entry_name}"
+  else
+    rdoc_system_path = File.expand_path(RI::Paths::SYSDIR)
+    if path.index(rdoc_system_path)
+      rel_path = path[/#{Regexp.escape(rdoc_system_path)}\/(.*)/, 1]
+      puts "Found in system  #{FastRI::Util.gem_relpath_to_full_name(rel_path)}"
+    else
+      puts "Found in #{path}:"
+    end
+  end
+  text = results.map do |result|
+    context = result.context(120)
+    from = (context.rindex("\n", context.index(result.query)) || -1) + 1
+    to   = (context.index("\n", context.index(result.query)) || 0) - 1
+    context_wrap(context[from..to], width)
+  end
+  puts
+  puts text.uniq[0...MAX_CONTEXT_LINES]
+  puts
+end
+
+if options[:do_full_text]
+  fulltext = File.join(options[:full_text_dir], "full_text.dat")
+  suffixes = File.join(options[:full_text_dir], "suffixes.dat")
+  begin
+    index = FastRI::FullTextIndex.new_from_filenames(fulltext, suffixes)
+  rescue Exception
+    puts <<EOF
+Couldn't open the full-text index:
+#{fulltext}
+#{suffixes}
+
+The index needs to be rebuilt with
+   fastri-server -B
+EOF
+    exit(-1)
+  end
+  gem_dir_info = FastRI::Util.gem_directories_unique
+  match_sets = ARGV.map do |query|
+    result = index.lookup(query)
+    if result
+      index.next_matches(result) + [result]
+    else
+      []
+    end
+  end
+  path_map = Hash.new{|h,k| h[k] = []}
+  match_sets.each{|matches| matches.each{|m| path_map[m.path] << m} }
+  paths = match_sets[1..-1].inject(match_sets[0].map{|x| x.path}.uniq) do |s,x|
+    s & x.map{|y| y.path}.uniq
+  end
+  if paths.empty?
+    puts "nil"
+  else
+    puts "#{paths.size} hits"
+    paths.sort_by{|path| -path_map[path].size}.map do |path|
+      puts "=" * options[:width]
+      display_fulltext_search_results(path_map[path], gem_dir_info, options[:width])
+    end
+  end
+  
+  exit 0
+end
+
 if options[:show_matches]
-  puts service.matches(ARGV[0], info_options).sort
+  puts service.matches(help_query, info_options).sort
 else
-  puts service.info(ARGV[0], info_options)
+  puts service.info(help_query, info_options)
 end
 # vi: set sw=2 expandtab:

data/bin/ri-emacs

@@ -1,4 +1,4 @@
-#! /home/batsman/usr//bin/ruby
+#!/usr/bin/env ruby
 ## drop-in replacement for the ri-emacs helper script for use 
 #  with ri-ruby.el, using the FastRI service via DRb
 #

data/lib/fastri/full_text_index.rb

@@ -0,0 +1,245 @@
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'fastri/full_text_indexer'
+require 'stringio'
+
+module FastRI
+
+class FullTextIndex
+  MAX_QUERY_SIZE = 20
+  MAX_REGEXP_MATCH_SIZE = 255
+  class Result
+    attr_reader :path, :query, :index, :metadata
+
+    def initialize(searcher, query, index, path, metadata)
+      @searcher = searcher
+      @index    = index
+      @query    = query
+      @path     = path
+      @metadata = metadata
+    end
+
+    def context(size)
+      @searcher.fetch_data(@index, 2*size+1, -size)
+    end
+
+    def text(size)
+      @searcher.fetch_data(@index, size, 0)
+    end
+  end
+
+  class << self; private :new end
+
+  DEFAULT_OPTIONS = {
+    :max_query_size => MAX_QUERY_SIZE,
+  }
+  
+  def self.new_from_ios(fulltext_IO, suffix_arrray_IO, options = {})
+    new(:io, fulltext_IO, suffix_arrray_IO, options)
+  end
+
+  def self.new_from_filenames(fulltext_fname, suffix_arrray_fname, options = {})
+    new(:filenames, fulltext_fname, suffix_arrray_fname, options)
+  end
+
+  attr_reader :max_query_size
+  def initialize(type, fulltext, sarray, options)
+    options = DEFAULT_OPTIONS.merge(options)
+    case type
+    when :io
+      @fulltext_IO = fulltext
+      @sarray_IO   = sarray
+    when :filenames
+      @fulltext_fname = fulltext
+      @sarray_fname   = sarray
+    else raise "Unknown type"
+    end
+    @type = type
+    @max_query_size = options[:max_query_size]
+    check_magic
+  end
+
+  def lookup(term)
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        case sarrayIO
+        when StringIO
+          num_suffixes = sarrayIO.string.size / 4 - 1
+        else
+          num_suffixes = sarrayIO.stat.size / 4 - 1
+        end
+
+        index, offset = binary_search(sarrayIO, fulltextIO, term, 0, num_suffixes)
+        if offset
+          fulltextIO.pos = offset
+          path, metadata = find_metadata(fulltextIO)
+          return Result.new(self, term, index, path, metadata) if path
+        else
+          nil
+        end
+      end
+    end
+  end
+
+  def next_match(result, term_or_regexp = "")
+    case term_or_regexp
+    when String;  size = [result.query.size, term_or_regexp.size].max
+    when Regexp;  size = MAX_REGEXP_MATCH_SIZE
+    end
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        idx = result.index
+        loop do
+          idx += 1
+          str = get_string(sarrayIO, fulltextIO, idx, size)
+          upto = str.index("\0")
+          str = str[0, upto] if upto
+          break unless str.index(result.query) == 0
+          if str[term_or_regexp]
+            fulltextIO.pos = index_to_offset(sarrayIO, idx)
+            path, metadata = find_metadata(fulltextIO)
+            return Result.new(self, result.query, idx, path, metadata) if path
+          end
+        end
+      end
+    end
+  end
+
+  def next_matches(result, term_or_regexp = "")
+    case term_or_regexp
+    when String;  size = [result.query.size, term_or_regexp.size].max
+    when Regexp;  size = MAX_REGEXP_MATCH_SIZE
+    end
+    ret = []
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        idx = result.index
+        loop do
+          idx += 1
+          str = get_string(sarrayIO, fulltextIO, idx, size)
+          upto = str.index("\0")
+          str = str[0, upto] if upto
+          break unless str.index(result.query) == 0
+          if str[term_or_regexp]
+            fulltextIO.pos = index_to_offset(sarrayIO, idx)
+            path, metadata = find_metadata(fulltextIO)
+            ret << Result.new(self, result.query, idx, path, metadata) if path
+          end
+        end
+      end
+    end
+
+    ret
+  end
+
+  def fetch_data(index, size, offset = 0)
+    raise "Bad offset" unless offset <= 0
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        base = index_to_offset(sarrayIO, index)
+        actual_offset = offset
+        newsize = size
+        if base + offset < 0    # at the beginning
+          excess        = (base + offset).abs   # remember offset is < 0
+          newsize       = size - excess
+          actual_offset = offset + excess
+        end
+        str  = get_string(sarrayIO, fulltextIO, index, newsize, offset)
+        from = (str.rindex("\0", -actual_offset) || -1) + 1
+        to   = (str.index("\0", -actual_offset) || 0) - 1
+        str[from..to]
+      end
+    end
+  end
+
+  private
+  def check_magic
+    get_fulltext_IO do |io|
+      io.rewind
+      header = io.read(FullTextIndexer::MAGIC.size)
+      raise "Unsupported index format." unless header
+      version = header[/\d+\.\d+\.\d+/]
+      raise "Unsupported index format." unless version
+      major, minor, teeny = version.scan(/\d+/)
+      if major != FASTRI_FT_INDEX_FORMAT_MAJOR or
+         minor > FASTRI_FT_INDEX_FORMAT_MINOR
+         raise "Unsupported index format"
+      end
+    end
+  end
+
+  def get_fulltext_IO
+    case @type
+    when :io; yield @fulltext_IO
+    when :filenames
+      File.open(@fulltext_fname, "rb"){|f| yield f}
+    end
+  end
+
+  def get_sarray_IO
+    case @type
+    when :io; yield @sarray_IO
+    when :filenames
+      File.open(@sarray_fname, "rb"){|f| yield f}
+    end
+  end
+
+  def index_to_offset(sarrayIO, index)
+    sarrayIO.pos = index * 4
+    sarrayIO.read(4).unpack("V")[0]
+  end
+
+  def find_metadata(fulltextIO)
+    oldtext = ""
+    loop do
+      text = fulltextIO.read(4096)
+      break unless text
+      if idx = text.index("\0")
+        if idx + 4 >= text.size
+          text.concat(fulltextIO.read(4096))
+        end
+        len     = text[idx+1, 4].unpack("V")[0]
+        missing = idx + 5 + len - text.size
+        if missing > 0
+          text.concat(fulltextIO.read(missing))
+        end
+        footer         = text[idx + 5, len - 1]
+        path, metadata = /(.*?)\0(.*)/m.match(footer).captures
+        return [path, Marshal.load(metadata)]
+      end
+      oldtext = text
+    end
+    nil
+  end
+
+  def get_string(sarrayIO, fulltextIO, index, size, off = 0)
+    sarrayIO.pos = index * 4
+    offset = sarrayIO.read(4).unpack("V")[0]
+    fulltextIO.pos = [offset + off, 0].max
+    fulltextIO.read(size)
+  end
+
+  def binary_search(sarrayIO, fulltextIO, term, from, to)
+    #puts "BINARY   #{from}  --  #{to}"
+    #left   = get_string(sarrayIO, fulltextIO, from, @max_query_size)
+    #right  = get_string(sarrayIO, fulltextIO, to, @max_query_size)
+    #puts "   #{left.inspect}  --  #{right.inspect}"
+    middle = (from + to) / 2
+    pivot = get_string(sarrayIO, fulltextIO, middle, @max_query_size)
+    if from == to
+      if pivot.index(term) == 0
+        sarrayIO.pos = middle * 4
+        [middle, sarrayIO.read(4).unpack("V")[0]]
+      else
+        nil
+      end
+    elsif term <= pivot
+      binary_search(sarrayIO, fulltextIO, term, from, middle)
+    elsif term > pivot
+      binary_search(sarrayIO, fulltextIO, term, middle+1, to)
+    end
+  end
+end # class FullTextIndex
+
+end # module FastRI