ferret 0.9.6 → 0.10.0

data/lib/ferret/utils/string_helper.rb

@@ -1,47 +0,0 @@
-module Ferret::Utils
-  module StringHelper 
-  # Methods for manipulating strings.
-
-    class StringReader
-      attr_reader :length
-
-      def initialize(str)
-        @str = str
-        @pointer = 0
-        @length = @str.length
-      end
-
-      def read(len = nil)
-        return @str if len.nil?
-
-        return nil if @pointer > @length
-
-        res = @str[@pointer, len]
-        @pointer += len
-        return res
-      end
-
-      def reset() @pointer = 0 end
-
-      def close() str = nil end
-    end
-
-    # Compares two strings, character by character, and returns the
-    # first position where the two strings differ from one another.
-    # eg.
-    #   string_difference('dustbin', 'dusty') # => 4
-    #   string_difference('dustbin', 'evening') # => 0
-    #   string_difference('eve', 'evening') # => 3
-    # 
-    # s1:: The first string to compare
-    # s2:: The second string to compare
-    # returns:: The first position where the two strings differ.
-    def StringHelper.string_difference(s1, s2) 
-      len = [s1.length, s2.length].min
-      len.times do |i|
-        return i if (s1[i] != s2[i]) 
-      end
-      return len
-    end
-  end
-end

data/lib/ferret/utils/thread_local.rb

@@ -1,28 +0,0 @@
-require 'thread'
-class Thread
-  def make_deleter
-    lambda{|id| @ferret_cache.delete(id)}
-  end
-
-  # Set the local value for the thread
-  def set_local(key, value)
-    @del ||= make_deleter
-    @ferret_cache ||= {}
-    ObjectSpace.define_finalizer(key, @del)
-    @ferret_cache[key.object_id] = value
-  end
-
-  # Get the local value for the thread
-  def get_local(key)
-    return (@ferret_cache ||= {})[key.object_id]
-  end
-
-  # Returns the number of local variables stored. Useful for testing.
-  def local_size
-    return (@ferret_cache ||= {}).size
-  end
-
-  def clear_local
-    (@ferret_cache ||= {}).clear
-  end
-end

data/lib/ferret/utils/weak_key_hash.rb

@@ -1,60 +0,0 @@
-module Ferret::Utils
-
-  require 'weakref'
-  require 'monitor'
-
-  # This class implements a weak key hash. ie all keys that are stored in this
-  # hash can still be garbage collected, and if they are garbage collected
-  # then the key and it's corresponding value will be deleted from the hash.
-  #    eg.
-  #    name = "david"
-  #    last_names = WeakKeyHash.new()
-  #    last_names[name] = "balmain"
-  #    puts last_names["david"]  #=>"balmain"
-  #    GC.start
-  #    puts last_names["david"]  #=>"balmain"
-  #    name = nil
-  #    GC.start
-  #    # the name "david" will now have been garbage collected so it should
-  #    # have been removed from the hash
-  #    puts last_names["david"]  #=>nil 
-  #
-  # WeakKeyHash subclasses Monitor so it can be synchronized on.
-  #
-  # === NOTE
-  # Unfortunately the ruby garbage collector is not always predictable so your
-  # results may differ but each key should eventually be freed when all other
-  # references have been removed and the garbage collector is ready.
-  class WeakKeyHash < Monitor
-    # Create a new WeakKeyHash.
-    def initialize
-      super()
-      @hash = {}
-      @deleter = lambda{|id| @hash.delete(id)}
-    end
-
-    # Set the value for the key just like a Hash
-    def []=(key, value)
-      ObjectSpace.define_finalizer(key, @deleter)
-      @hash[key.object_id] = value
-    end
-
-    # Get the value for the key
-    def [](key)
-      return @hash[key.object_id]
-    end
-
-    # Return the number of elements in the Hash
-    def size
-      @hash.size
-    end
-
-    # Print a string representation the WeakKeyHash
-    def to_s
-      buffer = ""
-      @hash.each_pair {|key, value| buffer << "<#{ObjectSpace._id2ref(key)}=>#{value}>"}
-      return buffer
-    end
-
-  end
-end

data/lib/rferret.rb

@@ -1,37 +0,0 @@
-$: << File.dirname(__FILE__)
-#--
-# Copyright (c) 2005 David Balmain
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-# :include: ../TUTORIAL
-module Ferret
-  VERSION = '0.9.6'
-end
-
-$ferret_pure_ruby = true
-require 'ferret/utils'
-require 'ferret/document'
-require 'ferret/stemmers'
-require 'ferret/analysis'
-require 'ferret/store'
-require 'ferret/index'
-require 'ferret/search'
-require 'ferret/query_parser'

data/rake_utils/code_statistics.rb

@@ -1,106 +0,0 @@
-# This code was taken from rails and is under the same license as ferret.
-# Thanks go to David Heinemeier Hansson
-class CodeStatistics
-  def initialize(*pairs)
-    @pairs      = pairs
-    @statistics = calculate_statistics
-    @total      = calculate_total if pairs.length > 1
-  end
-
-  def to_s
-    print_header
-    @pairs.each { |pair| print_line(pair.first, @statistics[pair.first]) }
-    print_splitter
-  
-    if @total
-      print_line("Total", @total)
-      print_splitter
-    end
-
-    print_code_test_stats
-  end
-
-  private
-    def calculate_statistics
-      @pairs.inject({}) { |stats, pair| stats[pair.first] = calculate_directory_statistics(pair.last); stats }
-    end
-
-    def calculate_directory_statistics(directory, pattern = /.*\.rb$/)
-      stats = { "lines" => 0, "codelines" => 0, "classes" => 0, "methods" => 0 }
-
-      Dir.foreach(directory) do |file_name| 
-        if File.stat(directory + "/" + file_name).directory? and (/^\./ !~ file_name)
-          newstats = calculate_directory_statistics(directory + "/" + file_name, pattern)
-          stats.each { |k, v| stats[k] += newstats[k] }
-        end
-
-        next unless file_name =~ pattern
-
-        f = File.open(directory + "/" + file_name)
-
-        while line = f.gets
-          stats["lines"]     += 1
-          stats["classes"]   += 1 if line =~ /class [A-Z]/
-          stats["methods"]   += 1 if line =~ /def [a-z]/
-          stats["codelines"] += 1 unless line =~ /^\s*$/ || line =~ /^\s*#/
-        end
-      end
-
-      stats
-    end
-
-    def calculate_total
-      total = { "lines" => 0, "codelines" => 0, "classes" => 0, "methods" => 0 }
-      @statistics.each_value { |pair| pair.each { |k, v| total[k] += v } }
-      total
-    end
-
-    def calculate_code
-      code_loc = 0
-      @statistics.each { |k, v| code_loc += v['codelines'] unless ['Units', 'Functionals'].include? k }
-      code_loc
-    end
-
-    def calculate_tests
-      test_loc = 0
-      @statistics.each { |k, v| test_loc += v['codelines'] if ['Units', 'Functionals'].include? k }
-      test_loc
-    end
-
-    def print_header
-      print_splitter
-      puts "| Name                 | Lines |   LOC | Classes | Methods | M/C | LOC/M |"
-      print_splitter
-    end
-
-    def print_splitter
-      puts "+----------------------+-------+-------+---------+---------+-----+-------+"
-    end
-
-    def print_line(name, statistics)
-      m_over_c   = (statistics["methods"] / statistics["classes"])   rescue m_over_c = 0
-      loc_over_m = (statistics["codelines"] / statistics["methods"]) - 2 rescue loc_over_m = 0
-
-      start = if ['Units', 'Functionals'].include? name
-        "|   #{name.ljust(18)} "
-      else
-        "| #{name.ljust(20)} " 
-      end
-
-      puts start + 
-           "| #{statistics["lines"].to_s.rjust(5)} " +
-           "| #{statistics["codelines"].to_s.rjust(5)} " +
-           "| #{statistics["classes"].to_s.rjust(7)} " +
-           "| #{statistics["methods"].to_s.rjust(7)} " +
-           "| #{m_over_c.to_s.rjust(3)} " +
-           "| #{loc_over_m.to_s.rjust(5)} |"
-    end
-
-    def print_code_test_stats
-      code  = calculate_code
-      tests = calculate_tests
-
-      puts "  Code LOC: #{code}     Test LOC: #{tests}     Code to Test Ratio: 1:#{sprintf("%.1f", tests.to_f/code)}"
-      puts ""
-    end
-  end

data/test/benchmark/tb_ram_store.rb

@@ -1,76 +0,0 @@
-require File.dirname(__FILE__) + "/../test_helper"
-require 'benchmark'
-
-class RAMStoreTest < Test::Unit::TestCase
-  def setup
-    @dir = Ferret::Store::RAMDirectory.new
-  end
-
-  def teardown
-    @dir.close()
-  end
-
-  def test_rw_bytes
-    bytes = [0x34, 0x87, 0xF9, 0xEA, 0x00, 0xFF]
-    rw_test(bytes, "byte")
-  end
-
-  def test_rw_ints
-    ints = [-2147483648, 2147483647, -1, 0]
-    rw_test(ints, "int")
-  end
-
-  def test_rw_longs
-    longs = [-9223372036854775808, 9223372036854775807, -1, 0]
-    rw_test(longs, "long")
-  end
-
-  def test_rw_uints
-    uints = [0xffffffff, 100000, 0]
-    rw_test(uints, "uint")
-  end
-
-  def test_rw_ulongs
-    ulongs = [0xffffffffffffffff, 100000000000000, 0]
-    rw_test(ulongs, "ulong")
-  end
-
-  def test_rw_vints
-    vints = [ 0xF8DC843342FE3484234987FE98AB987C897D214D123D123458EFBE2E238BACDEB9878790ABCDEF123DEF23988B89C,
-              0x0000000000000000000000000000000000000000,
-              0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF]
-    rw_test(vints, "vint")
-  end
-
-  def test_rw_vlongs
-    vlongs = [ 0xF8DC843342FE3484234987FE98AB987C897D214D123D123458EFBE2E238BACDEB9878790ABCDEF123DEF23988B89C,
-              0x0000000000000000000000000000000000000000,
-              0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF]
-    rw_test(vlongs, "vlong")
-  end
-
-  def test_rw_strings
-    strings = ['This is a ruby ferret test string ~!@#$%^&*()`123456790-=\)_+|', 'This is another string. I\'ll make this one a little longer than the last one. But I guess we need a few shorter ones too.', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten']
-    rw_test(strings, "string")
-  end
-
-  # this test fills up the output stream so that the buffer will have to be
-  # written a few times. It then uses seek to make sure that it works
-  # correctly
-
-  def rw_test(values, type)
-    puts "\nrw_#{type} test"
-    Benchmark.bmbm do |x|
-      x.report("write") do
-        ostream = @dir.create_output("rw_#{type}.test")
-        1000.times {values.each { |b| ostream.__send__("write_" + type, b) }}
-        ostream.close
-      end
-      x.report("read") do
-        istream = @dir.open_input("rw_#{type}.test")
-        1000.times {values.each { |b| assert_equal(b, istream.__send__("read_" + type), "#{type} should be equal") }}
-        istream.close
-      end
-    end
-  end
-end

data/test/benchmark/tb_rw_vint.rb

@@ -1,26 +0,0 @@
-$:.unshift File.join(File.dirname(__FILE__), '../../lib')
-
-require 'ferret'
-
-vints = [ 9223372036854775807,
-          0x00,
-          0xFFFFFFFFFFFFFFFF]
-t = Time.new
-10.times do
-  dpath = File.join(File.dirname(__FILE__),
-                       'fsdir')
-  dir = Ferret::Store::FSDirectory.new(dpath, true)
-
-  100.times do
-    ostream = dir.create_output("rw_vint.test")
-    300.times { |i| ostream.write_vint(vints[i%3]) }
-    ostream.close
-    istream = dir.open_input("rw_vint.test")
-    300.times { istream.read_vint }
-    istream.close
-  end
-
-  dir.close
-end
-
-puts "took #{Time.new - t} seconds"

data/test/functional/thread_safety_index_test.rb

@@ -1,81 +0,0 @@
-require File.dirname(__FILE__) + "/../test_helper"
-require File.dirname(__FILE__) + "/../utils/number_to_spoken.rb"
-require 'thread'
-
-class IndexThreadSafetyTest < Test::Unit::TestCase
-  include Ferret::Index
-  include Ferret::Document
-
-  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
-  ITERATIONS = 100
-  NUM_THREADS = 10
-  ANALYZER = Ferret::Analysis::Analyzer.new()
-
-  def setup
-    @index = Index.new(:path => 'index2',
-                       :create => true,
-                       :analyzer => ANALYZER,
-                       :default_field => 'contents')
-  end
-   
-  def indexing_thread()
-    ITERATIONS.times do
-      choice = rand()
-
-      if choice > 0.98
-        do_optimize
-      elsif choice > 0.9
-        do_delete_doc
-      elsif choice > 0.7
-        do_search
-      else
-        do_add_doc
-      end
-    end
-  rescue => e
-    puts e
-    puts e.backtrace
-    @index = nil
-    raise e
-  end 
-
-  def do_optimize
-    puts "Optimizing the index"
-    @index.optimize
-  end
-
-  def do_delete_doc
-    return if @index.size == 0
-    doc_num = rand(@index.size)
-    puts "Deleting #{doc_num} from index which has#{@index.has_deletions? ? "" : " no"} deletions"
-    puts "document was already deleted" if (@index.deleted?(doc_num))
-    @index.delete(doc_num)
-  end
-
-  def do_add_doc
-    d = Document.new()
-    n = rand(0xFFFFFFFF)
-    d << Field.new("id", n.to_s, Field::Store::YES, Field::Index::UNTOKENIZED)
-    d << Field.new("contents", n.to_spoken, Field::Store::NO, Field::Index::TOKENIZED)
-    puts("Adding #{n}")
-    @index << d
-  end
-  
-  def do_search
-    n = rand(0xFFFFFFFF)
-    puts("Searching for #{n}")
-    hits = @index.search_each(n.to_spoken, :num_docs => 3) do |d, s|
-      puts "Hit for #{n}: #{@index[d]["id"]} - #{s}"
-    end
-    puts("Searched for #{n}: total = #{hits}")
-  end
-
-  def test_threading
-    threads = []
-    NUM_THREADS.times do
-      threads << Thread.new { indexing_thread }
-    end
-
-    threads.each {|t| t.join}
-  end
-end