ferret 0.1.3 → 0.1.4

data/lib/ferret/store/ram_store.rb

@@ -99,7 +99,7 @@ module Ferret::Store
 
     def to_s
       str = "The files in this directory are: \n"
-      @files.each do |path,file|
+      @files.each do |path, file|
         str << path + " - " + file.size.to_s + "\n"
       end
       str
@@ -230,12 +230,12 @@ module Ferret::Store
       attr_accessor :mtime
       #attr_accessor :name
       attr_accessor :length
+      alias :size :length
 
 
       def initialize(name)
         @buffers = Array.new
         @mtime = Time.now
-        #@name = name
         @length = 0
       end
     end

data/lib/ferret/utils.rb

@@ -5,3 +5,4 @@ require 'ferret/utils/bit_vector'
 require 'ferret/utils/date_tools'
 require 'ferret/utils/number_tools'
 require 'ferret/utils/weak_key_hash'
+require 'ferret/utils/thread_local'

data/lib/ferret/utils/bit_vector.rb

@@ -74,7 +74,7 @@ module Ferret::Utils
     def write(d, name)
       output = d.create_output(name)
       begin 
-        output.write_vint(@bits)
+        output.write_string(self.class.bignum_to_string(@bits))
       ensure 
         output.close()
       end
@@ -86,7 +86,7 @@ module Ferret::Utils
       bv = BitVector.new
       input = d.open_input(name)
       begin 
-        bv.bits = input.read_vint()
+        bv.bits = string_to_bignum(input.read_string())
       ensure 
         input.close()
       end
@@ -101,5 +101,23 @@ module Ferret::Utils
       end
       puts ""
     end
+
+    # converts a BigNum into a string
+    def BitVector.bignum_to_string(num)
+      str = []
+      while (num > 0)
+        str << (num & 0xff)
+        num >>= 8
+      end
+      return str.pack("C*")
+    end
+
+    # converts a string into a bignum
+    def BitVector.string_to_bignum(str)
+      str = str.unpack("C*") 
+      num = 0
+      str.reverse.each {|c| num = ((num << 8) | c) }
+      return num
+    end
   end
 end

data/lib/ferret/utils/thread_local.rb

@@ -0,0 +1,28 @@
+require 'thread'
+class Thread
+  def make_deleter
+    lambda{|id| @ferret_cache.delete(id)}
+  end
+
+  # Set the local value for the thread
+  def set_local(key, value)
+    @del ||= make_deleter
+    @ferret_cache ||= {}
+    ObjectSpace.define_finalizer(key, @del)
+    @ferret_cache[key.object_id] = value
+  end
+
+  # Get the local value for the thread
+  def get_local(key)
+    return (@ferret_cache ||= {})[key.object_id]
+  end
+
+  # Returns the number of local variables stored. Useful for testing.
+  def local_size
+    return (@ferret_cache ||= {}).size
+  end
+
+  def clear_local
+    (@ferret_cache ||= {}).clear
+  end
+end

data/lib/ferret/utils/weak_key_hash.rb

@@ -1,6 +1,7 @@
 module Ferret::Utils
 
   require 'weakref'
+  require 'monitor'
 
   # This class implements a weak key hash. ie all keys that are stored in this
   # hash can still be garbage collected, and if they are garbage collected
@@ -18,29 +19,37 @@ module Ferret::Utils
   #    # have been removed from the hash
   #    puts last_names["david"]  #=>nil 
   #
-  # ===NOTE
+  # WeakKeyHash subclasses Monitor so it can be synchronized on.
+  #
+  # === NOTE
   # Unfortunately the ruby garbage collector is not always predictable so your
   # results may differ but each key should eventually be freed when all other
   # references have been removed and the garbage collector is ready.
-  class WeakKeyHash
+  class WeakKeyHash < Monitor
+    # Create a new WeakKeyHash.
     def initialize
+      super()
       @hash = {}
       @deleter = lambda{|id| @hash.delete(id)}
     end
 
+    # Set the value for the key just like a Hash
     def []=(key, value)
       ObjectSpace.define_finalizer(key, @deleter)
       @hash[key.object_id] = value
     end
 
+    # Get the value for the key
     def [](key)
       return @hash[key.object_id]
     end
 
+    # Return the number of elements in the Hash
     def size
       @hash.size
     end
 
+    # Print a string representation the WeakKeyHash
     def to_s
       buffer = ""
       @hash.each_pair {|key, value| buffer << "<#{ObjectSpace._id2ref(key)}=>#{value}>"}

data/test/benchmark/tb_rw_vint.rb

@@ -9,7 +9,7 @@ t = Time.new
 10.times do
   dpath = File.join(File.dirname(__FILE__),
                        'fsdir')
-  dir = Ferret::Store::FSDirectory.get_directory(dpath, true)
+  dir = Ferret::Store::FSDirectory.new(dpath, true)
 
   100.times do
     ostream = dir.create_output("rw_vint.test")

data/test/functional/thread_safety_index_test.rb

@@ -0,0 +1,81 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require File.dirname(__FILE__) + "/../utils/number_to_spoken.rb"
+require 'thread'
+
+class IndexThreadSafetyTest < Test::Unit::TestCase
+  include Ferret::Index
+  include Ferret::Document
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ITERATIONS = 100000
+  NUM_THREADS = 2
+  ANALYZER = Ferret::Analysis::Analyzer.new()
+
+  def setup
+    @index = Index.new(:path => 'index2',
+                       :create => true,
+                       :analyzer => ANALYZER,
+                       :default_field => 'contents')
+  end
+   
+  def indexing_thread()
+    ITERATIONS.times do
+      choice = rand()
+
+      if choice > 0.98
+        do_optimize
+      elsif choice > 0.9
+        do_delete_doc
+      elsif choice > 0.7
+        do_search
+      else
+        do_add_doc
+      end
+    end
+  rescue => e
+    puts e
+    puts e.backtrace
+    @index = nil
+    raise e
+  end 
+
+  def do_optimize
+    puts "Optimizing the index"
+    @index.optimize
+  end
+
+  def do_delete_doc
+    return if @index.size == 0
+    doc_num = rand(@index.size)
+    puts "Deleting #{doc_num} from index which has#{@index.has_deletions? ? "" : " no"} deletions"
+    puts "document was already deleted" if (@index.deleted?(doc_num))
+    @index.delete(doc_num)
+  end
+
+  def do_add_doc
+    d = Document.new()
+    n = rand(0xFFFFFFFF)
+    d << Field.new("id", n.to_s, Field::Store::YES, Field::Index::UNTOKENIZED)
+    d << Field.new("contents", n.to_spoken, Field::Store::NO, Field::Index::TOKENIZED)
+    puts("Adding #{n}")
+    @index << d
+  end
+  
+  def do_search
+    n = rand(0xFFFFFFFF)
+    puts("Searching for #{n}")
+    hits = @index.search_each(n.to_spoken, :num_docs => 3) do |d, s|
+      puts "Hit for #{n}: #{@index[d]["id"]} - #{s}"
+    end
+    puts("Searched for #{n}: total = #{hits}")
+  end
+
+  def test_threading
+    threads = []
+    NUM_THREADS.times do
+      threads << Thread.new { indexing_thread }
+    end
+
+    threads.each {|t| t.join}
+  end
+end

data/test/functional/thread_safety_test.rb

@@ -0,0 +1,137 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require File.dirname(__FILE__) + "/../utils/number_to_spoken.rb"
+require 'thread'
+
+class ThreadSafetyTest
+  include Ferret::Index
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Document
+
+  def initialize(options)
+    @options = options
+  end
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ANALYZER = Ferret::Analysis::Analyzer.new()
+  ITERATIONS = 19
+  @@searcher = nil
+  
+  def run_index_thread(writer)
+    reopen_interval = 30 + rand(60)
+
+    use_compound_file = false
+    
+    (400*ITERATIONS).times do |i|
+      d = Document.new()
+      n = rand(0xFFFFFFFF)
+      d << Field.new("id", n.to_s, Field::Store::YES, Field::Index::UNTOKENIZED)
+      d << Field.new("contents", n.to_spoken, Field::Store::NO, Field::Index::TOKENIZED)
+      puts("Adding #{n}")
+      
+      # Switch between single and multiple file segments
+      use_compound_file = (rand < 0.5)
+      writer.use_compound_file = use_compound_file
+      
+      writer << d
+
+      if (i % reopen_interval == 0) 
+        writer.close()
+        writer = IndexWriter.new(INDEX_DIR, :analyzer => ANALYZER)
+      end
+    end
+    
+    writer.close()
+  rescue => e
+    puts e
+    puts e.backtrace
+    raise e
+  end
+
+  def run_search_thread(use_global)
+    reopen_interval = 10 + rand(20)
+
+    unless use_global
+      searcher = IndexSearcher.new(INDEX_DIR)
+    end
+
+    (50*ITERATIONS).times do |i|
+      search_for(rand(0xFFFFFFFF), (searcher.nil? ? @@searcher : searcher))
+      if (i%reopen_interval == 0) 
+        if (searcher == nil) 
+          @@searcher = IndexSearcher.new(INDEX_DIR)
+        else 
+          searcher.close()
+          searcher = IndexSearcher.new(INDEX_DIR)
+        end
+      end
+    end
+  rescue => e
+    puts e
+    puts e.backtrace
+    raise e
+  end
+
+  def search_for(n, searcher)
+    puts("Searching for #{n}")
+    hits =
+      searcher.search(Ferret::QueryParser.parse(n.to_spoken, "contents", :analyzer => ANALYZER),
+                      :num_docs => 3)
+    puts("Search for #{n}: total = #{hits.size}")
+    hits.each do |d, s|
+      puts "Hit for #{n}: #{searcher.reader.get_document(d)["id"]} - #{s}"
+    end
+  end
+
+  def run_test_threads
+
+    threads = []
+    unless @options[:read_only]
+      writer = IndexWriter.new(INDEX_DIR, :analyzer => ANALYZER,
+                               :create => !@options[:add])
+      
+      threads << Thread.new { run_index_thread(writer) }
+      
+      sleep(1)
+    end
+      
+    threads << Thread.new { run_search_thread(false)}
+
+    @@searcher = IndexSearcher.new(INDEX_DIR) 
+    threads << Thread.new { run_search_thread(true)}
+
+    threads << Thread.new { run_search_thread(true)}
+
+    threads.each {|t| t.join}
+  end
+end
+
+
+if $0 == __FILE__
+  require 'optparse'
+
+  OPTIONS = {
+    :all        => false,
+    :read_only  => false,
+  }
+
+  ARGV.options do |opts|
+    script_name = File.basename($0)
+    opts.banner = "Usage: ruby #{script_name} [options]"
+
+    opts.separator ""
+
+    opts.on("-r", "--read-only", "Read Only.") { OPTIONS[:all] = true }
+    opts.on("-a", "--all", "All.") { OPTIONS[:read_only] = true }
+
+    opts.separator ""
+
+    opts.on("-h", "--help",
+            "Show this help message.") { puts opts; exit }
+
+    opts.parse!
+  end
+
+  tst = ThreadSafetyTest.new(OPTIONS)
+  tst.run_test_threads
+end

data/test/test_all.rb

@@ -1,9 +1,5 @@
 $:.unshift File.dirname(__FILE__)
+require 'test_helper.rb'
 
-require 'unit/ts_analysis.rb'
-require 'unit/ts_document.rb'
-require 'unit/ts_index.rb'
-require 'unit/ts_query_parser.rb'
-require 'unit/ts_search.rb'
-require 'unit/ts_store.rb'
-require 'unit/ts_utils.rb'
+load_test_dir("unit")
+#load_test_dir("functional")

data/test/test_helper.rb

@@ -1,9 +1,10 @@
+$:.unshift File.dirname(__FILE__)
 $:.unshift File.join(File.dirname(__FILE__), '../lib')
 $:.unshift File.join(File.dirname(__FILE__), '../ext')
 
 require 'test/unit'
 require 'ferret'
-require 'test/unit/index/th_doc'
+require 'unit/index/th_doc'
 
 def load_test_dir(dir)
   dir = File.join(File.dirname(__FILE__), dir)

data/test/unit/index/tc_compound_file_io.rb

@@ -60,8 +60,8 @@ class CompoundFileReaderTest < Test::Unit::TestCase
     cfile.close()
     
     cfile_reader = CompoundFileReader.new(@dir, "cfile")
-    assert_equal(4, cfile_reader.file_length('file1'))
-    assert_equal(15, cfile_reader.file_length('file2'))
+    assert_equal(4, cfile_reader.length('file1'))
+    assert_equal(15, cfile_reader.length('file2'))
     file1 = cfile_reader.open_input('file1')
     file2 = cfile_reader.open_input('file2')
     assert_equal(20, file1.read_int())

data/test/unit/index/tc_index.rb

@@ -4,6 +4,7 @@ require File.dirname(__FILE__) + "/../../test_helper"
 class IndexTest < Test::Unit::TestCase
   include Ferret::Index
   include Ferret::Analysis
+  include Ferret::Store
 
   def setup()
     @qp = Ferret::QueryParser.new()
@@ -134,12 +135,12 @@ class IndexTest < Test::Unit::TestCase
   end
 
   def test_fs_index
-    fs_path = File.join(File.dirname(__FILE__), '../../temp/fsdir')
-    `rm -rf #{fs_path}`
+    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+    `rm -rf #{File.join(fs_path, "*")}`
     assert_raise(Errno::ENOENT) {Index.new(:path => fs_path, :create_if_missing => false, :default_field => "def_field")}
     index = Index.new(:path => fs_path, :default_field => "def_field")
     do_test_index_with_array(index)
-    `rm -rf #{fs_path}`
+    `rm -rf #{File.join(fs_path, "*")}`
     index = Index.new(:path => fs_path, :create => true, :default_field => "def_field")
     do_test_index_with_hash(index)
     index = Index.new(:path => fs_path, :create => true, :default_field => "def_field")
@@ -147,8 +148,8 @@ class IndexTest < Test::Unit::TestCase
   end
 
   def test_fs_index_is_persistant
-    fs_path = File.join(File.dirname(__FILE__), '../../temp/fsdir')
-    `rm -rf #{fs_path}`
+    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+    `rm -rf #{File.join(fs_path, "*")}`
     data = [
       {"def_field" => "one two", :id => "me"},
       {"def_field" => "one", :field2 => "three"},
@@ -163,8 +164,129 @@ class IndexTest < Test::Unit::TestCase
     data.each {|doc| index << doc }
     assert_equal(8, index.size)
     index.close
-    index = Index.new(:path => fs_path, :default_field => "def_field")
+    index = Index.new(:path => fs_path, :create_if_missing => false)
     assert_equal(8, index.size)
     assert_equal("four", index[5]["field3"])
   end
+
+  def test_merging_indexes
+    data = [
+      {"f" => "zero"},
+      {"f" => "one"},
+      {"f" => "two"}
+    ]
+    index1 = Index.new(:default_field => "f")
+    data.each {|doc| index1 << doc }
+    data = [
+      {"f" => "three"},
+      {"f" => "four"},
+      {"f" => "five"}
+    ]
+    index2 = Index.new(:default_field => "f")
+    data.each {|doc| index2 << doc }
+    data = [
+      {"f" => "six"},
+      {"f" => "seven"},
+      {"f" => "eight"}
+    ]
+    index3 = Index.new(:default_field => "f")
+    data.each {|doc| index3 << doc }
+
+    index = Index.new(:default_field => "f")
+    index.add_indexes(index1)
+    assert_equal(3, index.size)
+    assert_equal("zero", index[0]["f"])
+    index.add_indexes([index2, index3])
+    assert_equal(9, index.size)
+    assert_equal("zero", index[0]["f"])
+    assert_equal("eight", index[8]["f"])
+    index1.close
+    index2.close
+    index3.close
+    assert_equal("seven", index[7]["f"])
+    data = [
+      {"f" => "alpha"},
+      {"f" => "beta"},
+      {"f" => "charlie"}
+    ]
+    dir1 = RAMDirectory.new
+    index1 = Index.new(:dir => dir1, :default_field => "f")
+    data.each {|doc| index1 << doc }
+    index1.flush
+    data = [
+      {"f" => "delta"},
+      {"f" => "echo"},
+      {"f" => "foxtrot"}
+    ]
+    dir2 = RAMDirectory.new
+    index2 = Index.new(:dir => dir2, :default_field => "f")
+    data.each {|doc| index2 << doc }
+    index2.flush
+    data = [
+      {"f" => "golf"},
+      {"f" => "india"},
+      {"f" => "juliet"}
+    ]
+    dir3 = RAMDirectory.new
+    index3 = Index.new(:dir => dir3, :default_field => "f")
+    data.each {|doc| index3 << doc }
+    index3.flush
+
+    index.add_indexes(dir1)
+    assert_equal(12, index.size)
+    assert_equal("alpha", index[9]["f"])
+    index.add_indexes([dir2, dir3])
+    assert_equal(18, index.size)
+    assert_equal("juliet", index[17]["f"])
+    index1.close
+    dir1.close
+    index2.close
+    dir2.close
+    index3.close
+    dir3.close
+    assert_equal("golf", index[15]["f"])
+  end
+
+  def test_persist_index
+    data = [
+      {"f" => "zero"},
+      {"f" => "one"},
+      {"f" => "two"}
+    ]
+    index = Index.new(:default_field => "f")
+    data.each {|doc| index << doc }
+    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+    index.persist(fs_path, true)
+    assert_equal(3, index.size)
+    assert_equal("zero", index[0]["f"])
+    index.close
+
+    index = Index.new(:path => fs_path)
+    assert_equal(3, index.size)
+    assert_equal("zero", index[0]["f"])
+    index.close
+
+
+    data = [
+      {"f" => "romeo"},
+      {"f" => "sierra"},
+      {"f" => "tango"}
+    ]
+    index = Index.new(:default_field => "f")
+    data.each {|doc| index << doc }
+    assert_equal(3, index.size)
+    assert_equal("romeo", index[0]["f"])
+    dir = FSDirectory.new(fs_path, false)
+    index.persist(dir)
+    assert_equal(6, index.size)
+    assert_equal("zero", index[0]["f"])
+    assert_equal("romeo", index[3]["f"])
+    index.close
+
+    index = Index.new(:path => fs_path)
+    assert_equal(6, index.size)
+    assert_equal("zero", index[0]["f"])
+    assert_equal("romeo", index[3]["f"])
+    index.close
+  end
 end