jk-ferret 0.11.8.2

data/test/long_running/largefile/tc_largefile.rb

@@ -0,0 +1,46 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SampleLargeTest < Test::Unit::TestCase
+  include Ferret::Index
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Utils
+  
+  INDEX_DIR = File.dirname(__FILE__) + "/../../temp/largefile"
+  RECORDS = 750
+  RECORD_SIZE = 10e5
+  
+  def setup
+    @index = Index.new(:path => INDEX_DIR, :create_if_missing => true, :key => :id)
+    create_index! if @index.size == 0 or ENV["RELOAD_LARGE_INDEX"]
+  end
+
+  def test_file_index_created
+    assert @index.size == RECORDS, "Index size should be #{RECORDS}, is #{@index.size}"
+  end
+  
+  def test_keys_work
+    @index << {:content => "foo", :id => RECORDS - 4}
+    assert @index.size == RECORDS, "Index size should be #{RECORDS}, is #{@index.size}"
+  end
+  
+  def test_read_file_after_two_gigs
+    assert @index.reader[RECORDS - 5].load.is_a?(Hash)
+  end
+  
+  def create_index!
+    @@already_built_large_index ||= false
+    return if @@already_built_large_index
+    @@already_built_large_index = true
+    a = "a"
+    RECORDS.times { |i|
+      seq = (a.succ! + " ") * RECORD_SIZE
+      record = {:id => i, :content => seq}
+    	@index << record
+    	print "i"
+    	STDOUT.flush
+    }
+    puts "o"
+    @index.optimize
+  end
+end

data/test/test_all.rb

@@ -0,0 +1,5 @@
+$:.unshift File.dirname(__FILE__)
+require 'test_helper.rb'
+
+load_test_dir("unit")
+#load_test_dir("functional")

data/test/test_helper.rb

@@ -0,0 +1,29 @@
+$:.unshift File.dirname(__FILE__)
+if $test_installed_gem
+  require 'rubygems'
+  require 'ferret'
+else
+  $:.unshift File.join(File.dirname(__FILE__), '../lib')
+  $:.unshift File.join(File.dirname(__FILE__), '../ext')
+end
+
+ENV['LANG'] = "en_US.UTF-8"
+ENV['LC_CTYPE'] = "en_US.UTF-8"
+
+class Float
+  def approx_eql?(o)
+    return (1 - self/o).abs < 0.0001
+  end
+  alias :=~ :approx_eql?
+end
+
+require 'test/unit'
+require 'ferret'
+require 'unit/index/th_doc' if (defined?(IndexTestHelper).nil?)
+
+
+def load_test_dir(dir)
+  Dir[File.join(File.dirname(__FILE__), dir, "t[scm]*.rb")].each do |file|
+    require file
+  end
+end

data/test/test_installed.rb

@@ -0,0 +1 @@
+$test_installed_gem = true

data/test/threading/number_to_spoken.rb

@@ -0,0 +1,132 @@
+# Author: Matthew D Moss
+#
+# Writtern for ruby quiz #25
+#
+class JapaneseTranslator
+    # My knowledge of counting Japanese is limited, so this may not
+    # be entirely correct; in particular, I don't know what rules
+    # to follow after 'hyaku man' (1,000,000).
+    # I also combine a digit with its group, such as 'gohyaku' rather
+    # than 'go hyaku'; I just like reading it better that way.
+
+    DIGITS = %w(zero ichi ni san yon go roku nana hachi kyu)
+    GROUPS = %w(nothingtoseeheremovealong ju hyaku sen)
+    MAN = 10000
+
+    def to_spoken(val)
+        case val <=> 0
+        when -1
+            '- ' + to_spoken(-val)
+        when 0
+            DIGITS[0]
+        else
+            group(val, 0)
+        end
+    end
+
+    private
+
+    def group(val, level)
+        if val >= MAN
+            group(val / MAN, 0) + 'man ' + group(val % MAN, 0)
+        else
+            case val
+            when 0
+                ''
+            when 1
+                level == 0 ? DIGITS[val] : GROUPS[level]
+            when 2...10
+                DIGITS[val] + (GROUPS[level] if level > 0).to_s
+            else
+                group(val / 10, level+1) + ' ' + group(val % 10, level)
+            end
+        end
+    end
+end
+
+
+class USEnglishTranslator
+    # Formal, US English. Optional 'and'. Will not produce things
+    # such as 'twelve hundred' but rather 'one thousand two hundred'.
+    # The use of 'and' is incomplete; it is sometimes missed.
+
+    DIGITS = %w(zero one two three four five six seven eight nine)
+    TEENS  = %w(ten eleven twelve thirteen fourteen fifteen sixteen
+                seventeen eighteen nineteen)
+    TENS   = %w(hello world twenty thirty forty fifty sixty seventy
+                eighty ninety)
+    GROUPS = %w(thousand million billion trillion quadrillion
+                quintillion sextillion septillion octillion nonillion
+                decillion)
+    K = 1000
+
+    def initialize(conjunction = true)
+        @conjunction = conjunction
+    end
+
+    def to_spoken(val)
+        case val <=> 0
+        when -1
+            'negative ' + to_spoken(-val)
+        when 0
+            DIGITS[0]
+        else
+            group(val, 0).flatten.join(' ')
+        end
+    end
+
+    private
+
+    def group(val, level)
+        x = group(val / K, level + 1) << GROUPS[level] if val >= K
+        x.to_a << under_1000(val % K, level)
+    end
+
+    def under_1000(val, level)
+        x = [DIGITS[val / 100]] << 'hundred' if val >= 100
+        x.to_a << under_100(val % 100, (level == 0 and not x.nil?))
+    end
+
+    def under_100(val, junction)
+        x = [('and' if @conjunction and junction)]    # wyf?
+        case val
+        when 0
+            []
+        when 1...10
+            x << DIGITS[val]
+        when 10...20
+            x << TEENS[val - 10]
+        else
+            d = val % 10
+            x << (TENS[val / 10] + ('-' + DIGITS[d] if d != 0).to_s)
+        end
+    end
+end
+
+
+class Integer
+    def to_spoken(translator = USEnglishTranslator.new)
+        translator.to_spoken(self).squeeze(' ').strip
+    end
+end
+
+if $0 == __FILE__
+    SAMPLES = [ 0, 1, 2, 5, 10, 11, 14, 18, 20, 21, 29, 33, 42, 50, 87, 99,
+                100, 101, 110, 167, 199, 200, 201, 276, 300, 314, 500, 610,
+                1000, 1039, 1347, 2309, 3098, 23501, 32767, 70000, 5480283,
+                2435489238, 234100090000, -42, -2001 ]
+
+    TRANSLATORS = { 'US English' => USEnglishTranslator.new,
+                    'Japanese'   => JapaneseTranslator.new }
+
+
+    # main
+    TRANSLATORS.each do |lang, translator|
+        puts
+        puts lang
+        puts '-' * lang.length
+        SAMPLES.each do |val|
+            puts "%12d => %s" % [val, val.to_spoken(translator)]
+        end
+    end
+end

data/test/threading/thread_safety_index_test.rb

@@ -0,0 +1,88 @@
+$:.unshift('.')
+require 'monitor'
+require File.dirname(__FILE__) + "/../test_helper"
+require File.dirname(__FILE__) + "/number_to_spoken.rb"
+require 'thread'
+
+class IndexThreadSafetyTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ITERATIONS = 100
+  NUM_THREADS = 3
+  ANALYZER = Ferret::Analysis::StandardAnalyzer.new()
+
+  def setup
+    index = Index.new(:path => INDEX_DIR,
+                      :create => true,
+                      :analyzer => ANALYZER,
+                      :default_field => :content)
+    index.close
+  end
+
+  def indexing_thread()
+    index = Index.new(:path => INDEX_DIR,
+                      :analyzer => ANALYZER,
+                      :auto_flush => true,
+                      :default_field => :content)
+
+    ITERATIONS.times do
+      choice = rand()
+
+      if choice > 0.98
+        do_optimize(index)
+      elsif choice > 0.7
+        do_delete_doc(index)
+      elsif choice > 0.5
+        do_search(index)
+      else
+        do_add_doc(index)
+      end
+      index.commit
+    end
+  rescue Exception => e
+    puts e
+    puts e.backtrace
+    raise 'hell'
+  end 
+
+  def do_optimize(index)
+    puts "Optimizing the index"
+    index.optimize
+  end
+
+  def do_delete_doc(index)
+    return if index.size == 0
+    doc_num = rand(index.size)
+    puts "Deleting #{doc_num} from index which has#{index.has_deletions? ? "" : " no"} deletions"
+    puts "document was already deleted" if (index.deleted?(doc_num))
+    index.delete(doc_num)
+  end
+
+  def do_add_doc(index)
+    n = rand(0xFFFFFFFF)
+    d = {:id => n, :content => n.to_spoken}
+    puts("Adding #{n}")
+    index << d
+  end
+  
+  def do_search(index)
+    n = rand(0xFFFFFFFF)
+    puts("Searching for #{n}")
+    hits = index.search_each(n.to_spoken, :num_docs => 3) do |d, s|
+      puts "Hit for #{n}: #{index[d][:id]} - #{s}"
+    end
+    puts("Searched for #{n}: total = #{hits}")
+  end
+
+  def test_threading
+    threads = []
+    NUM_THREADS.times do
+      threads << Thread.new { indexing_thread }
+    end
+
+    threads.each {|t| 
+      t.join
+    }
+  end
+end

data/test/threading/thread_safety_read_write_test.rb

@@ -0,0 +1,73 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require File.dirname(__FILE__) + "/number_to_spoken.rb"
+require 'thread'
+
+class IndexThreadSafetyReadWriteTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ITERATIONS = 10000
+  ANALYZER = Ferret::Analysis::Analyzer.new()
+
+  def setup
+    @index = Index.new(:path => INDEX_DIR,
+                       :create => true,
+                       :analyzer => ANALYZER,
+                       :default_field => :content)
+  end
+
+  def search_thread()
+    ITERATIONS.times do
+      do_search()
+      sleep(rand(1))
+    end
+  rescue => e
+    puts e
+    puts e.backtrace
+    @index = nil
+    raise e
+  end 
+
+  def index_thread()
+    ITERATIONS.times do
+      do_add_doc()
+      sleep(rand(1))
+    end
+  rescue => e
+    puts e
+    puts e.backtrace
+    @index = nil
+    raise e
+  end 
+
+  def do_add_doc
+    n = rand(0xFFFFFFFF)
+    d = {:id => n.to_s, :content => n.to_spoken}
+    puts("Adding #{n}")
+    begin
+      @index << d
+    rescue => e
+      puts e
+      puts e.backtrace
+      @index = nil
+      raise e
+    end
+  end
+  
+  def do_search
+    n = rand(0xFFFFFFFF)
+    puts("Searching for #{n}")
+    hits = @index.search_each(n.to_spoken, :num_docs => 3) do |d, s|
+      puts "Hit for #{n}: #{@index[d]["id"]} - #{s}"
+    end
+    puts("Searched for #{n}: total = #{hits}")
+  end
+
+  def test_threading
+    threads = []
+    threads << Thread.new { search_thread }
+    threads << Thread.new { index_thread }
+
+    threads.each { |t| t.join }
+  end
+end

data/test/threading/thread_safety_test.rb

@@ -0,0 +1,133 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require File.join(File.dirname(__FILE__), "number_to_spoken.rb")
+require 'thread'
+
+class ThreadSafetyTest
+  include Ferret::Index
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret
+
+  def initialize(options)
+    @options = options
+  end
+
+  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
+  ANALYZER = Ferret::Analysis::WhiteSpaceAnalyzer.new()
+  ITERATIONS = 1000
+  QUERY_PARSER = Ferret::QueryParser.new(:analyzer => ANALYZER,
+                                         :default_field => 'contents')
+  @@searcher = nil
+  
+  def run_index_thread(writer)
+    reopen_interval = 30 + rand(60)
+
+    use_compound_file = false
+    
+    (400*ITERATIONS).times do |i|
+      n = rand(0xFFFFFFFF)
+      d = {:id => n.to_s, :contents => n.to_spoken}
+      puts("Adding #{n}")
+      
+      # Switch between single and multiple file segments
+      use_compound_file = (rand < 0.5)
+      writer.use_compound_file = use_compound_file
+      
+      writer << d
+
+      if (i % reopen_interval == 0) 
+        writer.close()
+        writer = IndexWriter.new(:path => INDEX_DIR, :analyzer => ANALYZER)
+      end
+    end
+    
+    writer.close()
+  rescue => e
+    puts e
+    puts e.backtrace
+    raise e
+  end
+
+  def run_search_thread(use_global)
+    reopen_interval = 10 + rand(20)
+
+    unless use_global
+      searcher = Searcher.new(INDEX_DIR)
+    end
+
+    (50*ITERATIONS).times do |i|
+      search_for(rand(0xFFFFFFFF), (searcher.nil? ? @@searcher : searcher))
+      if (i%reopen_interval == 0) 
+        if (searcher == nil) 
+          @@searcher = Searcher.new(INDEX_DIR)
+        else 
+          searcher.close()
+          searcher = Searcher.new(INDEX_DIR)
+        end
+      end
+    end
+  rescue => e
+    puts e
+    puts e.backtrace
+    raise e
+  end
+
+  def search_for(n, searcher)
+    puts("Searching for #{n}")
+    topdocs = searcher.search(QUERY_PARSER.parse(n.to_spoken), :limit => 3)
+    puts("Search for #{n}: total = #{topdocs.total_hits}")
+    topdocs.hits.each do |hit|
+      puts "Hit for #{n}: #{searcher.reader[hit.doc]["id"]} - #{hit.score}"
+    end
+  end
+
+  def run_test_threads
+    threads = []
+    unless @options[:read_only]
+      writer = IndexWriter.new(:path => INDEX_DIR, :analyzer => ANALYZER,
+                               :create => !@options[:add])
+      
+      threads << Thread.new { run_index_thread(writer) }
+      sleep(1)
+    end
+      
+    threads << Thread.new { run_search_thread(false)}
+
+    @@searcher = Searcher.new(INDEX_DIR) 
+    threads << Thread.new { run_search_thread(true)}
+
+    threads << Thread.new { run_search_thread(true)}
+
+    threads.each {|t| t.join}
+  end
+end
+
+
+if $0 == __FILE__
+  require 'optparse'
+
+  OPTIONS = {
+    :all        => false,
+    :read_only  => false,
+  }
+
+  ARGV.options do |opts|
+    script_name = File.basename($0)
+    opts.banner = "Usage: ruby #{script_name} [options]"
+
+    opts.separator ""
+
+    opts.on("-r", "--read-only", "Read Only.") { OPTIONS[:all] = true }
+    opts.on("-a", "--all", "All.") { OPTIONS[:read_only] = true }
+
+    opts.separator ""
+
+    opts.on("-h", "--help",
+            "Show this help message.") { puts opts; exit }
+
+    opts.parse!
+  end
+
+  tst = ThreadSafetyTest.new(OPTIONS)
+  tst.run_test_threads
+end