ferret 0.3.2 → 0.9.0

data/lib/ferret/search/multi_phrase_query.rb

@@ -123,16 +123,17 @@ module Ferret::Search
         query_expl = Explanation.new()
         query_expl.description = "query_weight(#{@query}), product of:"
 
-        boost_expl = Explanation.new(@query.boost(), "boost")
-        (query_expl << boost_expl) if (@query.boost() != 1.0)
-
+        boost = @query.boost()
+        if boost != 1.0
+          boost_expl = Explanation.new(boost, "boost")
+          query_expl << boost_expl
+        end
         query_expl << idf_expl
         
         query_norm_expl = Explanation.new(@query_norm,"query_norm")
         query_expl << query_norm_expl
         
-        query_expl.value =
-          boost_expl.value * idf_expl.value * query_norm_expl.value
+        query_expl.value = boost * @idf * @query_norm
 
         result << query_expl
        

data/lib/ferret/search/phrase_query.rb

@@ -127,7 +127,7 @@ module Ferret::Search
         query_norm_expl = Explanation.new(@query_norm, "query_norm")
         query_expl << query_norm_expl
         
-        query_expl.value = boost * @idf * query_norm_expl.value
+        query_expl.value = boost * @idf * @query_norm
 
         result << query_expl
        
@@ -150,15 +150,12 @@ module Ferret::Search
         field_expl << field_norm_expl
 
         field_expl.value = tf_expl.value * @idf * field_norm
-        
         result << field_expl
 
-        # combine them
-        result.value = query_expl.value * field_expl.value
-
-        if query_expl.value == 1.0
+        if (query_expl.value == 1.0)
           return field_expl
         else
+          result.value = query_expl.value * field_expl.value
           return result
         end
       end

data/lib/ferret/search/prefix_query.rb

@@ -23,10 +23,10 @@ module Ferret::Search
             term.text[0,prefix_length] != prefix_text)
             break
           end
-            tq = TermQuery.new(term)                        # found a match
-            tq.boost = boost()                              # set the boost
-            bq.add_query(tq, BooleanClause::Occur::SHOULD)  # add to query
-            #puts("added " + term)
+          tq = TermQuery.new(term)                        # found a match
+          tq.boost = boost()                              # set the boost
+          bq.add_query(tq, BooleanClause::Occur::SHOULD)  # add to query
+          #puts("added " + term)
         end while (enumerator.next?)
       ensure 
         enumerator.close()

data/lib/ferret/search/sort.rb

@@ -87,8 +87,10 @@ module Ferret::Search
           SortField.new(field, {:sort_type => SortField::SortType::AUTO,
                                 :reverse => reverse})
         end
-        @fields << SortField::FIELD_DOC if @fields.size == 1
       end
+      doc_sort_added = false
+      @fields.each {|f| doc_sort_added = true if f == SortField::FIELD_DOC }
+      @fields << SortField::FIELD_DOC if not doc_sort_added
     end
 
     # Represents sorting by computed relevance. Using this sort criteria returns

data/lib/ferret/search/sort_field.rb

@@ -56,16 +56,16 @@ module Ferret::Search
     # name:: Name of field to sort by.  Can be +nil+ if +sort_type+ is SCORE or
     #     DOC.
     #
-    # A hash with the followind values can also be supplied;
-    # sort_type:: Type of values in the terms.
-    # reverse:: True if natural order should be reversed.
-    # comparator:: a proc used to compare two values from the index. You can
-    #    also give this value to the SortType object that you pass.
-    def initialize(name = nil, args= {})
+    # An options hash with the followind values can also be supplied;
+    # sort_type::  Type of values in the terms.
+    # reverse::    True if natural order should be reversed.
+    # comparator:: A proc used to compare two values from the index. You can
+    #              also give this value to the SortType object that you pass.
+    def initialize(name = nil, options= {})
       @name = name.to_s if name
-      @sort_type = args[:sort_type]||SortType::AUTO
-      @reverse = args[:reverse]||false
-      @comparator = args[:comparator]||@sort_type.comparator
+      @sort_type = options[:sort_type]||SortType::AUTO
+      @reverse = options[:reverse]||false
+      @comparator = options[:comparator]||@sort_type.comparator
       if (@name == nil and @sort_type != SortType::DOC and
           @sort_type != SortType::SCORE)
         raise ArgumentError, "You must supply a field name for your sort field"

data/lib/ferret/search/spans/near_spans_enum.rb

@@ -62,7 +62,7 @@ module Ferret::Search::Spans
           @length = finish() - start()      # compute new length
           @parent.total_length += @length   # add new length to total
 
-          if (@parent.max == nil or doc() > @parent.max.doc() or   # maintain max
+          if (@parent.max.nil? or doc() > @parent.max.doc() or   # maintain max
               (doc() == @parent.max.doc and finish() > @parent.max.finish))
             @parent.max = self
           end

data/lib/ferret/search/spans/span_near_query.rb

@@ -49,7 +49,7 @@ module Ferret::Search::Spans
     def to_s(field = nil) 
       buffer = "span_near(["
       buffer << @clauses.map {|c| c.to_s(field)}.join(", ")
-      buffer << "], #{@stop}, #{@in_order})"
+      buffer << "], #{@slop}, #{@in_order})"
       return buffer
     end
 

data/lib/ferret/search/spans/span_weight.rb

@@ -6,7 +6,7 @@ module Ferret::Search::Spans
       @query = query
       @terms = query.terms()
 
-      @idf = @query.similarity(searcher).idf_phrase(@terms, searcher)
+      @idf = @similarity.idf_phrase(@terms, searcher)
     end
 
     attr_reader :query, :value

data/lib/ferret/search/spans/spans_enum.rb

@@ -2,7 +2,7 @@ module Ferret::Search::Spans
   # Expert: an enumeration of span matches.  Used to implement span searching.
   # Each span represents a range of term positions within a document.  Matches
   # are enumerated in order, by increasing document number, within that by
-  # increasing start position and ensure by increasing finish position. 
+  # increasing start position and finally by increasing finish position. 
   class SpansEnum 
     # Move to the next match, returning true iff any such exists. 
     def next?()
@@ -13,12 +13,12 @@ module Ferret::Search::Spans
     # greater than or equal to _target_. Returns true iff there is such a
     # match.  Behaves as if written:
     #
-    #    def skip_to(target) 
-    #      begin 
-    #        return false if (!next?)
-    #      end while (target > doc)
-    #      return true
-    # end
+    #   def skip_to(target) 
+    #     begin 
+    #       return false if (!next?)
+    #     end while (target > doc)
+    #     return true
+    #   end
     #
     # Most implementations are considerably more efficient than that.
     def skip_to(target)

data/lib/ferret/store/fs_store.rb

@@ -38,7 +38,7 @@ module Ferret::Store
       super()
       if create then FileUtils.mkdir_p(path) end
       if not File.directory?(path) then
-        raise "There is no directory: #{path}. Use create = true to create one"
+        raise IOError, "There is no directory: #{path}. Use create = true to create one"
       end
       @dir = Dir.new(path)
       # put the lock_dir here as well if no default exists.
@@ -182,7 +182,7 @@ module Ferret::Store
 
     # Construct a Lock.
     def make_lock(name) 
-      FSLock.new(@lock_dir.path + "/" + lock_prefix() + name)
+      FSLock.new(@lock_dir.path + "/" + lock_prefix() + name + ".lck")
     end
 
     # Closes the store.
@@ -285,7 +285,11 @@ module Ferret::Store
       attr_reader   :length, :file
 
       def initialize(path)
-        @file = File.open(path, "rb")
+        begin
+          @file = File.open(path, "rb")
+        rescue Errno::ENOENT => e
+          raise StandardError.new(e.message)
+        end
         @file.extend(MonitorMixin)
         #class <<@file
         #  attr_accessor :ref_count
@@ -312,7 +316,7 @@ module Ferret::Store
       private
 
         def read_internal(b, offset, length)
-          @file.synchronize do
+          #@file.synchronize do
             position = pos()
             if position != @file.pos
               @file.seek(position)
@@ -322,7 +326,7 @@ module Ferret::Store
               raise EOFError, "Read past EOF in #{@file.path}"
             end
             b[offset, bytes.length] = bytes
-          end
+          #end
         end
 
         def seek_internal(pos)
@@ -340,7 +344,7 @@ module Ferret::Store
 
       # returns the lock prefix for this directory
       def lock_prefix
-        LOCK_PREFIX + Digest::MD5.hexdigest(@dir.path)
+        LOCK_PREFIX
       end
 
       # Unfortunately, on Windows, Dir does not refresh when rewind is called

data/lib/ferret/store/ram_store.rb

@@ -89,7 +89,7 @@ module Ferret::Store
 
     # Construct a Lock.
     def make_lock(name) 
-      RAMLock.new(LOCK_PREFIX + name, self)
+      RAMLock.new(LOCK_PREFIX + name + ".lck", self)
     end
 
 
@@ -252,14 +252,14 @@ module Ferret::Store
       # obtain the lock on the data source 
       def obtain(lock_timeout = 1) 
         MAX_ATTEMPTS.times do
-          @dir.synchronize do
+          #@dir.synchronize do
             # create a file if none exists. If one already exists
             # then someone beat us to the lock so return false
             if (! locked?) then
               @dir.create_output(@lock_file)
               return true
             end
-          end
+          #end
           # lock was not obtained so sleep for timeout then try again.
           sleep(lock_timeout)
         end

data/lib/rferret.rb

@@ -0,0 +1,36 @@
+#--
+# Copyright (c) 2005 David Balmain
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+# :include: ../TUTORIAL
+module Ferret
+  VERSION = '0.9.0'
+end
+
+$ferret_pure_ruby = true
+require 'ferret/utils'
+require 'ferret/document'
+require 'ferret/stemmers'
+require 'ferret/analysis'
+require 'ferret/store'
+require 'ferret/index'
+require 'ferret/search'
+require 'ferret/query_parser'

data/test/functional/thread_safety_index_test.rb

@@ -7,8 +7,8 @@ class IndexThreadSafetyTest < Test::Unit::TestCase
   include Ferret::Document
 
   INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
-  ITERATIONS = 100000
-  NUM_THREADS = 2
+  ITERATIONS = 100
+  NUM_THREADS = 10
   ANALYZER = Ferret::Analysis::Analyzer.new()
 
   def setup

data/test/test_helper.rb

@@ -2,13 +2,27 @@ $:.unshift File.dirname(__FILE__)
 $:.unshift File.join(File.dirname(__FILE__), '../lib')
 $:.unshift File.join(File.dirname(__FILE__), '../ext')
 
+class Float
+  def =~(o)
+    return (1 - self/o).abs < 0.00001
+  end
+end
+
 require 'test/unit'
-require 'ferret'
 require 'unit/index/th_doc'
+if $ferret_pure_ruby
+  require 'rferret'
+else
+  require 'ferret'
+end
 
 def load_test_dir(dir)
   dir = File.join(File.dirname(__FILE__), dir)
   Dir.foreach(dir) do |file|
-    require File.join(dir, file) if file =~ /^t[mcs]_.*\.rb$/
+    if $ferret_pure_ruby
+      require File.join(dir, file) if file =~ /^t?[mcs]_.*\.rb$/
+    else
+      require File.join(dir, file) if file =~ /^[mcs]_.*\.rb$/
+    end
   end
 end

data/test/unit/analysis/c_token.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class TokenTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_token()
+    tk1 = Token.new("DBalmain", 1, 8, 5, "token")
+    assert_equal(tk1, Token.new("DBalmain", 1, 8))
+    assert_not_equal(tk1, Token.new("DBalmain", 0, 8))
+    assert_not_equal(tk1, Token.new("DBalmain", 1, 9))
+    assert_not_equal(tk1, Token.new("Dbalmain", 1, 8))
+    assert(tk1 < Token.new("CBalmain", 2, 7))
+    assert(tk1 > Token.new("EBalmain", 0, 9))
+    assert(tk1 < Token.new("CBalmain", 1, 9))
+    assert(tk1 > Token.new("EBalmain", 1, 7))
+    assert(tk1 < Token.new("EBalmain", 1, 8))
+    assert(tk1 > Token.new("CBalmain", 1, 8))
+    assert_equal("DBalmain", tk1.text)
+    tk1.text = "Hello"
+    assert_equal("Hello", tk1.text)
+    assert_equal(1, tk1.start_offset)
+    assert_equal(8, tk1.end_offset)
+    assert_equal(5, tk1.pos_inc)
+  end
+end

data/test/unit/analysis/tc_per_field_analyzer_wrapper.rb

@@ -25,7 +25,7 @@ class PerFieldAnalyzerWrapperTest < Test::Unit::TestCase
     assert_equal(Token.new('My', 22, 24), t.next())
     assert_equal(Token.new('e-mail', 25, 31), t.next())
     assert_equal(Token.new("ADDRESS", 32, 39), t.next())
-    if ( token = t.next()): puts token.term_text end
+    if ( token = t.next()): puts token.text end
     assert(! t.next())
     input.reset()
     t = aw.token_stream("body", input)

data/test/unit/analysis/tc_standard_analyzer.rb

@@ -4,7 +4,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
   include Ferret::Utils::StringHelper
   include Ferret::Analysis
 
-  def test_lettertokenizer()
+  def test_standard_analyzer()
     input = StringReader.new('D.Ba_l-n@gma-l.com AB&Sons Toys\'r\'us you\'re she\'s, #$%^$%*& job@dot I.B.M. the an AnD THEIR')
     sa = StandardAnalyzer.new()
     t = sa.token_stream("field", input)

data/test/unit/document/c_field.rb

@@ -0,0 +1,98 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class FieldTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Utils
+
+  def test_store()
+    assert_not_nil(Field::Store::COMPRESS)
+    assert_not_nil(Field::Store::YES)
+    assert_not_nil(Field::Store::NO)
+  end
+
+  def test_index()
+    assert_not_nil(Field::Index::TOKENIZED)
+    assert_not_nil(Field::Index::UNTOKENIZED)
+    assert_not_nil(Field::Index::NO)
+    assert_not_nil(Field::Index::NO_NORMS)
+  end
+
+  def test_term_vector()
+    assert_not_nil(Field::TermVector::YES)
+    assert_not_nil(Field::TermVector::NO)
+    assert_not_nil(Field::TermVector::WITH_POSITIONS)
+    assert_not_nil(Field::TermVector::WITH_OFFSETS)
+    assert_not_nil(Field::TermVector::WITH_POSITIONS_OFFSETS)
+  end
+
+  def test_standard_field()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    assert_equal("name", f.name)
+    assert_equal("value", f.data)
+    assert_equal(true, f.stored?)
+    assert_equal(true, f.compressed?)
+    assert_equal(true, f.indexed?)
+    assert_equal(true, f.tokenized?)
+    assert_equal(false, f.store_term_vector?)
+    assert_equal(false, f.store_offsets?)
+    assert_equal(false, f.store_positions?)
+    assert_equal(false, f.omit_norms?)
+    assert_equal(false, f.binary?)
+    assert_equal("stored/compressed,indexed,tokenized,<name:value>", f.to_s)
+    f.data = "183"
+    f.boost = 0.001
+    assert_equal("183", f.data)
+    assert(0.001 =~ f.boost)
+  end
+
+  def test_set_store()
+    f = Field.new("name", "", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.store = Field::Store::NO
+    assert_equal(false, f.stored?)
+    assert_equal(false, f.compressed?)
+    assert_equal("indexed,tokenized,<name:>", f.to_s)
+  end
+
+  def test_set_index()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.index = Field::Index::NO
+    assert_equal(false, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(false, f.omit_norms?)
+    assert_equal("stored/compressed,<name:value>", f.to_s)
+    f.index = Field::Index::NO_NORMS
+    assert_equal(true, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(true, f.omit_norms?)
+    assert_equal("stored/compressed,indexed,omit_norms,<name:value>", f.to_s)
+  end
+
+  def test_set_term_vector()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.term_vector = Field::TermVector::WITH_POSITIONS_OFFSETS
+    assert_equal(true, f.store_term_vector?)
+    assert_equal(true, f.store_offsets?)
+    assert_equal(true, f.store_positions?)
+    assert_equal("stored/compressed,indexed,tokenized,store_term_vector,store_offsets,store_positions,<name:value>", f.to_s)
+  end
+
+  def test_new_binary_field()
+    tmp = []
+    256.times {|i| tmp[i] = i}
+    bin = tmp.pack("c*")
+    f = Field.new_binary_field("name", bin, Field::Store::YES)
+    assert_equal("name", f.name)
+    assert_equal(bin, f.data)
+    assert_equal(true, f.stored?)
+    assert_equal(false, f.compressed?)
+    assert_equal(false, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(false, f.store_term_vector?)
+    assert_equal(false, f.store_offsets?)
+    assert_equal(false, f.store_positions?)
+    assert_equal(false, f.omit_norms?)
+    assert_equal(true, f.binary?)
+    assert_equal("stored/uncompressed,binary,<name:=bin_data=>", f.to_s)
+  end
+end