dakrone-fastri 0.3.1

data/bin/ri-emacs

@@ -0,0 +1,202 @@
+#!/usr/bin/env ruby
+## drop-in replacement for the ri-emacs helper script for use 
+#  with ri-ruby.el, using the FastRI service via DRb
+#
+# Based on ri-emacs.rb by Kristof Bastiaensen <kristof@vleeuwen.org>
+#
+#    Copyright (C) 2004,2006 Kristof Bastiaensen
+#                  2006 Mauricio Fernandez <mfp@acm.org>
+#    
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#----------------------------------------------------------------------
+
+require 'rinda/ring'
+require 'optparse'
+require 'fastri/util'
+
+# {{{ cmdline parsing and service discovery
+# we bind to 127.0.0.1 by default, because otherwise Ruby will try with
+# 0.0.0.0, which results in a DNS request, adding way too much latency
+options = {:addr => "127.0.0.1", :width => ENV['RI_EMACS_COLUMNS'] ? ENV['RI_EMACS_COLUMNS'].to_i : 72}
+override_addr_env = false
+optparser = OptionParser.new do |opts|
+  opts.banner = "Usage: ri-emacs [options] <query>"
+
+  opts.on("-s", "--bind [ADDR]", "Bind to ADDR for incoming DRb connections.",
+          "(default: 127.0.0.1)") do |addr|
+    options[:addr] = addr
+    override_addr_env = true
+  end
+
+  opts.on("-w", "--width WIDTH", "Set the width of the output.") do |width|
+    options[:width] = width
+  end
+
+  opts.on("-h", "--help", "Show this help message") do 
+    puts opts
+    exit
+  end
+end
+optparser.parse!
+
+if override_addr_env
+  addr = "druby://#{options[:addr]}:0"
+else
+  addr =  "druby://#{ENV["FASTRI_ADDR"]||options[:addr]}:0"
+end
+
+begin
+  DRb.start_service(addr)
+  ring_server = Rinda::RingFinger.primary
+rescue Exception
+  puts <<EOF
+Couldn't initialize DRb and locate the Ring server.
+
+Please make sure that:
+ * the fastri-server is running, the server is bound to the correct interface,
+   and the ACL setup allows connections from this host
+ * fri is using the correct interface for incoming DRb requests:
+   either set the FASTRI_ADDR environment variable, or use --bind ADDR, e.g
+      export FASTRI_ADDR="192.168.1.12"
+      fri Array
+EOF
+  exit(-1)                      # '
+end
+service = ring_server.read([:name, :FastRI, nil, nil])[2]
+
+class EventLoop
+  include FastRI::Util::MagicHelp
+
+  def initialize(ri, options)
+    @ri = ri
+    @opts = options
+  end
+
+  def run
+    puts "READY"
+    loop do
+      line = $stdin.gets
+      cmd, p = /(\w+)(.*)$/.match(line)[1..2]
+      p.strip!
+      case cmd
+      when "TRY_COMPLETION";       puts complete_try(p)
+      when "COMPLETE_ALL";         puts complete_all(p)
+      when "LAMBDA";               puts complete_lambda(p)
+      when "CLASS_LIST";           puts class_list(p)
+      when "CLASS_LIST_WITH_FLAG"; puts class_list_with_flag(p)
+      when "DISPLAY_ARGS";         display_args(p)
+      when "DISPLAY_INFO";         display_info(p)
+      end
+    end
+  end
+
+  def complete_try(keyw)
+    list = @ri.completion_list(keyw)
+    if list.nil? 
+      return "nil"
+    elsif list.size == 1 and
+      list[0].split(/(::)|#|\./) == keyw.split(/(::)|#|\./)
+      return "t"
+    end
+
+    first = list.shift;
+    if first =~ /(.*)((?:::)|(?:#))(.*)/
+      other = $1 + ($2 == "::" ? "#" : "::") + $3
+    end
+
+    len = first.size
+    match_both = false
+    list.each do |w|
+      while w[0, len] != first[0, len]
+        if other and w[0, len] == other[0, len]
+          match_both = true
+          break
+        end
+        len -= 1
+      end
+    end
+
+    if match_both
+      return other.sub(/(.*)((?:::)|(?:#))/) { $1 + "." }[0, len].inspect
+    else
+      return first[0, len].inspect
+    end
+  end
+
+  def complete_all(keyw)
+    list = @ri.completion_list(keyw)
+    if list.nil?
+      "nil"
+    else
+      "(" + list.map { |w| w.inspect }.join(" ") + ")"
+    end
+  end
+
+  def complete_lambda(keyw)
+    list = @ri.completion_list(keyw)
+    if list.nil?
+      "nil"
+    else
+      if list.find { |n| n.split(/(::)|#|\./) == keyw.split(/(::)|#|\./) }
+        "t"
+      else
+        "nil"
+      end
+    end
+  end
+
+  def class_list(keyw)
+    list = @ri.class_list(keyw)
+    if list
+      "(" + list.map{|x| "(#{x.inspect})"}.join(" ") + ")"
+    else
+      "nil"
+    end
+  end
+
+  def class_list_with_flag(keyw)
+    list = @ri.class_list_with_flag(keyw)
+    if list
+      "(" + list.map{|x| "(#{x.inspect})"}.join(" ") + ")"
+    else
+      "nil"
+    end
+  end
+
+  def display_(what, keyw)
+    data = @ri.__send__(what, magic_help(keyw), :width => @opts[:width])
+    if data
+      puts data
+    elsif (new_keyw = FastRI::Util.change_query_method_type(keyw)) != keyw
+      puts @ri.__send__(what, new_keyw, :width => @opts[:width])
+    end
+    puts "RI_EMACS_END_OF_INFO"
+  end
+
+  def display_args(keyw)
+    display_ :args, keyw
+  end
+
+  def display_info(keyw)
+    display_ :info, keyw
+  end
+end
+
+
+#{{{ event loop
+#$stdout.sync = true # better not set sync=true, causes problems with emacs
+EventLoop.new(service, options).run
+
+# vi: set sw=2 expandtab:

data/fastri.gemspec

@@ -0,0 +1,64 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{fastri}
+  s.version = "0.3.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Mauricio Fernandez", "Lee Hinman"]
+  s.date = %q{2009-07-17}
+  s.default_executable = %q{fri}
+  s.description = %q{Fastri is RI, fast.}
+  s.email = %q{lee@writequit.org}
+  s.executables = ["fastri-server", "fri", "ri-emacs"]
+  s.extra_rdoc_files = [
+    "README.en"
+  ]
+  s.files = [
+    "fastri.gemspec",
+    "Rakefile",
+    "README.en",
+    "THANKS",
+    "CHANGES",
+    "COPYING",
+    "LEGAL",
+    "LICENSE",
+    "bin/fastri-server",
+    "bin/fri",
+    "bin/ri-emacs",
+    "lib/fastri/full_text_index.rb",
+    "lib/fastri/full_text_indexer.rb",
+    "lib/fastri/name_descriptor.rb",
+    "lib/fastri/ri_index.rb",
+    "lib/fastri/ri_service.rb",
+    "lib/fastri/util.rb",
+    "lib/fastri/version.rb",
+    "indexer.rb",
+    "lookup.rb",
+    "pre-install.rb",
+    "setup.rb",
+    "indexer.rb",
+    "test/test_full_text_index.rb",
+    "test/test_full_text_indexer.rb",
+    "test/test_functional_ri_service.rb",
+    "test/test_integration_full_text_index.rb",
+    "test/test_name_descriptor.rb",
+    "test/test_ri_index.rb",
+    "test/test_util.rb"
+  ]
+  s.homepage = %q{http://github.com/dakrone/fastri}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{0.3.1}
+  s.summary = %q{Fastri is RI, fast.}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/indexer.rb

@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+# Full-text indexing of the RI documentation.
+# This is the proof of concept that evolved into FastRI's full-text searching
+# subsystem.
+
+class IndexBuilder
+  MAXWORD_SIZE = 20
+  def initialize(fulltext_file, index_file)
+    @fulltext_file = fulltext_file
+    @index_file    = index_file
+    @fulltext      = ""
+  end
+
+  def add_document(name, contents)
+    @fulltext << preprocess(contents)
+    @fulltext << "\0#{name}\0"
+  end
+
+  require 'strscan'
+  require 'enumerator'
+  def finish
+    File.open(@fulltext_file, "w"){|f| f.puts @fulltext }
+    scanner = StringScanner.new(@fulltext)
+
+    count = 0
+    suffixes = []
+    until scanner.eos?
+      count += 1
+      if count == 100
+        print "%3d%%\r" % (100 * scanner.pos / @fulltext.size)
+        $stdout.flush
+        count = 0
+      end
+      start = scanner.pos
+      text = scanner.scan_until(/\0.*?\0/)
+      text = text.sub(/\0.*?\0$/,"")
+      suffixes.concat find_suffixes(text, start)
+      scanner.terminate if !text
+    end
+    puts "Suffixes: #{suffixes.size}"
+    t0 = Time.new
+    sorted = suffixes.sort_by{|x| @fulltext[x,MAXWORD_SIZE]}
+    File.open(@index_file, "w") do |f|
+      sorted.each_slice(10000){|x| f.write x.pack("V*")}
+    end
+    File.open("suffixes", "w"){|f| sorted.each{|i| f.puts @fulltext[i,MAXWORD_SIZE].inspect}}
+    puts "Processed in #{Time.new - t0} seconds"
+  end
+
+  require 'strscan'
+  def find_suffixes(string, offset)
+    suffixes = []
+    sc = StringScanner.new(string)
+    until sc.eos?
+      sc.skip(/([^A-Za-z_]|\n)*/)
+      len = string.size
+      loop do
+        break if sc.pos == len
+        suffixes << offset + sc.pos
+        break unless sc.skip(/[A-Za-z0-9_]+([^A-Za-z0-9_]|\n)*/)
+      end
+    end
+    suffixes
+  end
+
+  private
+  def preprocess(str)
+    str.gsub(/\0/,"")
+  end
+end
+
+def linearize(comment)
+  case s = comment["body"]
+  when String; s
+  else 
+    if Array === (y = comment["contents"])
+      y.map{|z| linearize(z)}.join("\n")
+    elsif s = comment["text"]
+      s
+    else
+      nil
+    end
+  end
+end
+
+require 'rdoc/ri/ri_paths'
+require 'yaml'
+$:.unshift "lib"
+require 'fastri/util'
+
+#paths = RI::Paths::PATH
+gem_paths = FastRI::Util.gem_directories_unique.map{|_,_,path| path}
+paths = [ RI::Paths::SYSDIR, RI::Paths::SITEDIR, RI::Paths::HOMEDIR ].find_all do |p|
+  p && File.directory?(p)
+end
+paths.concat gem_paths
+indexer = IndexBuilder.new("test_FULLTEXT", "test_INDEX")
+bad = 0
+paths.each do |path|
+  Dir["#{path}/**/*.yaml"].each do |yamlfile|
+    yaml = File.read(yamlfile)
+    begin
+      data = YAML.load(yaml.gsub(/ \!.*/, ''))
+    rescue Exception
+      bad += 1
+      puts "Couldn't load #{yamlfile}"
+      #puts "=" * 80
+      #puts yaml
+      next
+    end
+
+    desc = (data['comment']||[]).map{|x| linearize(x)}.join("\n")
+    desc.gsub!(/<\/?(em|b|tt|ul|ol|table)>/, "")
+    desc.gsub!(/&quot;/, "'")
+    desc.gsub!(/&lt;/, "<")
+    desc.gsub!(/&gt;/, ">")
+    desc.gsub!(/&amp;/, "&")
+=begin
+    puts "=" * 80
+    puts yamlfile
+    puts "-" * 80
+    puts yaml
+    puts "-" * 80
+    puts desc
+    $stdin.gets
+=end
+    unless desc.empty?
+      indexer.add_document(yamlfile, desc) 
+    end
+  end
+end
+puts "BAD files: #{bad}"
+indexer.finish

data/lib/fastri/full_text_index.rb

@@ -0,0 +1,245 @@
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'fastri/full_text_indexer'
+require 'stringio'
+
+module FastRI
+
+class FullTextIndex
+  MAX_QUERY_SIZE = 20
+  MAX_REGEXP_MATCH_SIZE = 255
+  class Result
+    attr_reader :path, :query, :index, :metadata
+
+    def initialize(searcher, query, index, path, metadata)
+      @searcher = searcher
+      @index    = index
+      @query    = query
+      @path     = path
+      @metadata = metadata
+    end
+
+    def context(size)
+      @searcher.fetch_data(@index, 2*size+1, -size)
+    end
+
+    def text(size)
+      @searcher.fetch_data(@index, size, 0)
+    end
+  end
+
+  class << self; private :new end
+
+  DEFAULT_OPTIONS = {
+    :max_query_size => MAX_QUERY_SIZE,
+  }
+  
+  def self.new_from_ios(fulltext_IO, suffix_arrray_IO, options = {})
+    new(:io, fulltext_IO, suffix_arrray_IO, options)
+  end
+
+  def self.new_from_filenames(fulltext_fname, suffix_arrray_fname, options = {})
+    new(:filenames, fulltext_fname, suffix_arrray_fname, options)
+  end
+
+  attr_reader :max_query_size
+  def initialize(type, fulltext, sarray, options)
+    options = DEFAULT_OPTIONS.merge(options)
+    case type
+    when :io
+      @fulltext_IO = fulltext
+      @sarray_IO   = sarray
+    when :filenames
+      @fulltext_fname = fulltext
+      @sarray_fname   = sarray
+    else raise "Unknown type"
+    end
+    @type = type
+    @max_query_size = options[:max_query_size]
+    check_magic
+  end
+
+  def lookup(term)
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        case sarrayIO
+        when StringIO
+          num_suffixes = sarrayIO.string.size / 4 - 1
+        else
+          num_suffixes = sarrayIO.stat.size / 4 - 1
+        end
+
+        index, offset = binary_search(sarrayIO, fulltextIO, term, 0, num_suffixes)
+        if offset
+          fulltextIO.pos = offset
+          path, metadata = find_metadata(fulltextIO)
+          return Result.new(self, term, index, path, metadata) if path
+        else
+          nil
+        end
+      end
+    end
+  end
+
+  def next_match(result, term_or_regexp = "")
+    case term_or_regexp
+    when String;  size = [result.query.size, term_or_regexp.size].max
+    when Regexp;  size = MAX_REGEXP_MATCH_SIZE
+    end
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        idx = result.index
+        loop do
+          idx += 1
+          str = get_string(sarrayIO, fulltextIO, idx, size)
+          upto = str.index("\0")
+          str = str[0, upto] if upto
+          break unless str.index(result.query) == 0
+          if str[term_or_regexp]
+            fulltextIO.pos = index_to_offset(sarrayIO, idx)
+            path, metadata = find_metadata(fulltextIO)
+            return Result.new(self, result.query, idx, path, metadata) if path
+          end
+        end
+      end
+    end
+  end
+
+  def next_matches(result, term_or_regexp = "")
+    case term_or_regexp
+    when String;  size = [result.query.size, term_or_regexp.size].max
+    when Regexp;  size = MAX_REGEXP_MATCH_SIZE
+    end
+    ret = []
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        idx = result.index
+        loop do
+          idx += 1
+          str = get_string(sarrayIO, fulltextIO, idx, size)
+          upto = str.index("\0")
+          str = str[0, upto] if upto
+          break unless str.index(result.query) == 0
+          if str[term_or_regexp]
+            fulltextIO.pos = index_to_offset(sarrayIO, idx)
+            path, metadata = find_metadata(fulltextIO)
+            ret << Result.new(self, result.query, idx, path, metadata) if path
+          end
+        end
+      end
+    end
+
+    ret
+  end
+
+  def fetch_data(index, size, offset = 0)
+    raise "Bad offset" unless offset <= 0
+    get_fulltext_IO do |fulltextIO|
+      get_sarray_IO do |sarrayIO|
+        base = index_to_offset(sarrayIO, index)
+        actual_offset = offset
+        newsize = size
+        if base + offset < 0    # at the beginning
+          excess        = (base + offset).abs   # remember offset is < 0
+          newsize       = size - excess
+          actual_offset = offset + excess
+        end
+        str  = get_string(sarrayIO, fulltextIO, index, newsize, offset)
+        from = (str.rindex("\0", -actual_offset) || -1) + 1
+        to   = (str.index("\0", -actual_offset) || 0) - 1
+        str[from..to]
+      end
+    end
+  end
+
+  private
+  def check_magic
+    get_fulltext_IO do |io|
+      io.rewind
+      header = io.read(FullTextIndexer::MAGIC.size)
+      raise "Unsupported index format." unless header
+      version = header[/\d+\.\d+\.\d+/]
+      raise "Unsupported index format." unless version
+      major, minor, teeny = version.scan(/\d+/)
+      if major != FASTRI_FT_INDEX_FORMAT_MAJOR or
+         minor > FASTRI_FT_INDEX_FORMAT_MINOR
+         raise "Unsupported index format"
+      end
+    end
+  end
+
+  def get_fulltext_IO
+    case @type
+    when :io; yield @fulltext_IO
+    when :filenames
+      File.open(@fulltext_fname, "rb"){|f| yield f}
+    end
+  end
+
+  def get_sarray_IO
+    case @type
+    when :io; yield @sarray_IO
+    when :filenames
+      File.open(@sarray_fname, "rb"){|f| yield f}
+    end
+  end
+
+  def index_to_offset(sarrayIO, index)
+    sarrayIO.pos = index * 4
+    sarrayIO.read(4).unpack("V")[0]
+  end
+
+  def find_metadata(fulltextIO)
+    oldtext = ""
+    loop do
+      text = fulltextIO.read(4096)
+      break unless text
+      if idx = text.index("\0")
+        if idx + 4 >= text.size
+          text.concat(fulltextIO.read(4096))
+        end
+        len     = text[idx+1, 4].unpack("V")[0]
+        missing = idx + 5 + len - text.size
+        if missing > 0
+          text.concat(fulltextIO.read(missing))
+        end
+        footer         = text[idx + 5, len - 1]
+        path, metadata = /(.*?)\0(.*)/m.match(footer).captures
+        return [path, Marshal.load(metadata)]
+      end
+      oldtext = text
+    end
+    nil
+  end
+
+  def get_string(sarrayIO, fulltextIO, index, size, off = 0)
+    sarrayIO.pos = index * 4
+    offset = sarrayIO.read(4).unpack("V")[0]
+    fulltextIO.pos = [offset + off, 0].max
+    fulltextIO.read(size)
+  end
+
+  def binary_search(sarrayIO, fulltextIO, term, from, to)
+    #puts "BINARY   #{from}  --  #{to}"
+    #left   = get_string(sarrayIO, fulltextIO, from, @max_query_size)
+    #right  = get_string(sarrayIO, fulltextIO, to, @max_query_size)
+    #puts "   #{left.inspect}  --  #{right.inspect}"
+    middle = (from + to) / 2
+    pivot = get_string(sarrayIO, fulltextIO, middle, @max_query_size)
+    if from == to
+      if pivot.index(term) == 0
+        sarrayIO.pos = middle * 4
+        [middle, sarrayIO.read(4).unpack("V")[0]]
+      else
+        nil
+      end
+    elsif term <= pivot
+      binary_search(sarrayIO, fulltextIO, term, from, middle)
+    elsif term > pivot
+      binary_search(sarrayIO, fulltextIO, term, middle+1, to)
+    end
+  end
+end # class FullTextIndex
+
+end # module FastRI