ferret 0.11.6 → 0.11.8.4

data/test/unit/search/tm_searcher.rb

@@ -34,6 +34,9 @@ module SearcherTests
     docs.length.times do |i|
       assert_equal(expected[i], docs[i].doc)
     end
+    if options[:limit] == :all and options[:offset] == nil
+      assert_equal(expected.sort, @searcher.scan(query))
+    end
   end
 
   def test_offset
@@ -201,6 +204,36 @@ module SearcherTests
     check_hits(rq, [15,16,17])
   end
 
+  def test_typed_range_query()
+    rq = TypedRangeQuery.new(:number, :>= => "-1.0", :<= => 1.0)
+    check_hits(rq, [0,1,4,10,15,17])
+
+    rq = TypedRangeQuery.new(:number, :>  => "-1.0", :<  => 1.0)
+    check_hits(rq, [0,1,4,15])
+
+    if ENV['FERRET_DEV']
+        # text hexadecimal
+        rq = TypedRangeQuery.new(:number, :>  =>  "1.0", :<= =>"0xa")
+        check_hits(rq, [6,7,9,12])
+    end
+
+    # test single bound
+    rq = TypedRangeQuery.new(:number, :<= =>  "0.0")
+    check_hits(rq, [5,11,15,16,17])
+
+    # test single bound
+    rq = TypedRangeQuery.new(:number, :>  =>  "0.0")
+    check_hits(rq, [0,1,2,3,4,6,7,8,9,10,12,13,14])
+
+    # below range - no results
+    rq = TypedRangeQuery.new(:number, :>  =>  "10051006", :<  =>"10051010")
+    check_hits(rq, [])
+
+    # above range - no results
+    rq = TypedRangeQuery.new(:number, :>  =>  "-12518421", :<  =>"-12518420")
+    check_hits(rq, [])
+  end
+
   def test_prefix_query()
     pq = PrefixQuery.new(:category, "cat1")
     check_hits(pq, [0, 1, 2, 3, 4, 13, 14, 15, 16, 17])
@@ -358,7 +391,6 @@ module SearcherTests
     assert_equal("<b>the words</b>...", highlights[0])
     assert_equal("...<b>one</b> <b>two</b>...", highlights[1])
 
-    # {:dates => '20070505, 20071230, 20060920, 20081111'},
     [
       [RangeQuery.new(:dates, :>= => '20081111'),
         '20070505 20071230 20060920 <b>20081111</b>'],
@@ -381,4 +413,24 @@ module SearcherTests
     #assert_equal("<b>the words</b>...", highlights[0])
     #assert_equal("...<b>one</b> <b>two</b>...", highlights[1])
   end
+
+  def test_highlighter_with_standard_analyzer()
+    dir = Ferret::Store::RAMDirectory.new
+    iw = Ferret::Index::IndexWriter.new(:dir => dir,
+                  :analyzer => Ferret::Analysis::StandardAnalyzer.new())
+    [
+        {:field => "field has a url http://ferret.davebalmain.com/trac/ end"},
+    ].each {|doc| iw << doc }
+    iw.close
+    
+    searcher = Searcher.new(dir)
+
+    q = TermQuery.new(:field, "ferret.davebalmain.com/trac");
+    highlights = searcher.highlight(q, 0, :field,
+                                    :excerpt_length => 1000,
+                                    :num_excerpts => 1)
+    assert_equal(1, highlights.size)
+    assert_equal("field has a url <b>http://ferret.davebalmain.com/trac/</b> end",
+                 highlights[0])
+  end
 end

data/test/unit/store/tc_fs_store.rb

@@ -2,6 +2,8 @@ require File.dirname(__FILE__) + "/../../test_helper"
 require File.dirname(__FILE__) + "/tm_store"
 require File.dirname(__FILE__) + "/tm_store_lock"
 
+require 'fileutils'
+
 class FSStoreTest < Test::Unit::TestCase
   include Ferret::Store
   include StoreTest
@@ -13,12 +15,12 @@ class FSStoreTest < Test::Unit::TestCase
   end
 
   def teardown
-    @dir.refresh()
     @dir.close()
+    Dir[File.join(@dpath, "*")].each {|path| begin File.delete(path) rescue nil end}
   end
 
   def test_fslock
-    lock_name = "lfile"
+    lock_name = "_file.f1"
     lock_file_path = make_lock_file_path(lock_name)
     assert(! File.exists?(lock_file_path), "There should be no lock file")
     lock = @dir.make_lock(lock_name)
@@ -63,6 +65,42 @@ class FSStoreTest < Test::Unit::TestCase
 #    assert(! File.exists?(lock_file_path), "The lock file should have been deleted")
 #  end
 #
+  def test_permissions
+    _S_IRGRP = 0040
+    _S_IWGRP = 0020
+
+    dpath = File.expand_path(File.join(File.dirname(__FILE__),
+                       '../../temp/fsdir_permissions'))
+
+    FileUtils.mkdir_p(dpath)
+    dstat = File.stat(dpath)
+
+    File.chown(nil, `id -G`.split.last.to_i, dpath)
+    File.chmod(dstat.mode | _S_IRGRP | _S_IWGRP, dpath)
+
+    dir = FSDirectory.new(dpath, true)
+
+    file_name = 'test_permissions'
+    file_path = File.join(dpath, file_name)
+
+    dir.touch(file_name)
+
+    mode = File.stat(file_path).mode
+
+    assert(mode & _S_IRGRP == _S_IRGRP, "file should be group-readable")
+    assert(mode & _S_IWGRP == _S_IWGRP, "file should be group-writable")
+  ensure
+    if dstat
+      File.chown(nil, dstat.gid, dpath)
+      File.chmod(dstat.mode, dpath)
+    end
+
+    if dir
+      dir.refresh()
+      dir.close()
+    end
+  end
+
   def make_lock_file_path(name)
     lock_file_path = File.join(@dpath, lfname(name))
     if File.exists?(lock_file_path) then

data/test/unit/store/tm_store_lock.rb

@@ -1,6 +1,5 @@
 module StoreLockTest
   class Switch
-    @@counter = 0
     def Switch.counter() return @@counter end
     def Switch.counter=(counter) @@counter = counter end
   end
@@ -14,7 +13,7 @@ module StoreLockTest
     assert(lock1.obtain(lock_time_out))
     assert(lock2.locked?)
 
-    assert(! can_obtain_lock?(lock2))
+    assert(! can_obtain_lock?(lock2, lock_time_out))
 
     exception_thrown = false
     begin
@@ -31,6 +30,8 @@ module StoreLockTest
     assert(lock2.obtain(lock_time_out))
     lock2.release()
 
+    Switch.counter = 0
+
     t = Thread.new() do
       lock1.while_locked(lock_time_out) do
         Switch.counter = 1
@@ -46,7 +47,8 @@ module StoreLockTest
     while Switch.counter < 1
     end
     
-    assert(! can_obtain_lock?(lock2))
+    assert(! can_obtain_lock?(lock2, lock_time_out),
+           "lock 2 should not be obtainable")
 
     Switch.counter = 2
     while Switch.counter < 3
@@ -56,12 +58,11 @@ module StoreLockTest
     lock2.release()
   end
 
-  def can_obtain_lock?(lock)
-    lock_time_out = 0.001 # we want this test to run quickly
+  def can_obtain_lock?(lock, lock_time_out)
     begin
       lock.obtain(lock_time_out)
       return true
-    rescue
+    rescue Exception=>e
     end
     return false
   end

data/test/unit/tc_field_symbol.rb

@@ -0,0 +1,26 @@
+require File.dirname(__FILE__) + "/../test_helper"
+
+class FieldSymbolTest < Test::Unit::TestCase
+  def test_field_symbol
+    Ferret::FIELD_TYPES.each do |field_type|
+      assert(:sym.respond_to?(field_type),
+             "Symbol doesn't respond to #{field_type}")
+    end
+
+    %w(desc desc? type).each do  |method|
+      assert(:sym.respond_to?(method),
+             "Symbol doesn't respond to #{method}")
+    end
+
+    assert_nil(:sym.type)
+    assert(!:sym.desc?)
+    assert(:sym.desc.desc?)
+    assert(!:sym.desc.desc.desc?)
+
+    Ferret::FIELD_TYPES.each do |field_type|
+      assert_equal(field_type, :sym.__send__(field_type).type)
+    end
+
+    assert(:string, :sym.integer.byte.float.string.type)
+  end
+end

data/test/utils/content_generator.rb

@@ -0,0 +1,226 @@
+module ContentGenerator
+  wpath = File.expand_path(File.join(__FILE__, '../../../data/words'))
+  WORDS = File.readlines(wpath).collect {|w| w.strip}
+  CHARS = 'abcdefghijklmnopqrstuvwxyz1234567890`~!@#$%^&*()_-+={[}]|\\:;"\'<,>.?/'
+  ALNUM = 'abcdefghijklmnopqrstuvwxyz1234567890'
+  ALPHA = 'abcdefghijklmnopqrstuvwxyz'
+  URL_SUFFIXES = %w{com net org biz info}
+  URL_COUNTRY_CODES = %w{au jp uk nz tv}
+  TEXT_CACHE = {}
+  WORD_CACHE = {}
+  MARKDOWN_EMPHASIS_MARKERS = %w{* _ ** __ ` ``}
+  MARKDOWN_LIST_MARKERS = %w{- * + 1.}
+
+  def self.generate_text(length = 5..10, options = {})
+    if length.is_a?(Range)
+      raise ArgumentError, "range must be positive" unless length.min
+      length = length.min + rand(length.max - length.min)
+    end
+
+    text = ''
+    if options[:chars]
+      while word = random_word and text.size + word.size < length
+        text << word + ' '
+      end
+      text.strip!
+      text << generate_word(length - text.size)
+    else
+      text = Array.new(length) {|x| random_word}.join(' ')
+    end
+    if key = options[:unique]||options[:key]
+      cache = TEXT_CACHE[key]||={}
+      if cache[text]
+        return generate_text(options)
+      else
+        return cache[text] = true
+      end      
+    end
+    return text
+  end
+
+  def self.generate_word(length = 5..10, options = {})
+    if length.is_a?(Range)
+      raise ArgumentError, "range must be positive" unless length.min
+      length = length.min + rand(length.max - length.min)
+    end
+
+    word = ''
+    case options[:charset]
+    when :alpha
+      word = Array.new(length) {|x| random_alpha}.pack('c*')
+    when :alnum
+      word = Array.new(length) {|x| random_alnum}.pack('c*')
+    else
+      word = Array.new(length) {|x| random_char}.pack('c*')
+    end
+
+    if key = options[:unique]||options[:key]
+      cache = WORD_CACHE[key]||={}
+      if cache[word]
+        return generate_word(options)
+      else
+        cache[word] = true
+      end      
+    end
+    return word
+  end
+
+  def self.generate_alpha_word(length = 5..10, options = {})
+    options[:charset] = :alpha
+    generate_word(length, options)
+  end
+
+  def self.generate_alnum_word(length = 5..10, options = {})
+    options[:charset] = :alnum
+    generate_word(length, options)
+  end
+
+  def self.generate_email(options = {})
+    num_name_sections = 1 + rand(2)
+    num_url_sections = 1 + rand(2)
+    name = Array.new(num_name_sections) {|x| generate_alnum_word }.join('.')
+    url = [generate_alnum_word]
+    url += Array.new(num_url_sections) {|x| generate_alpha_word(2..3) }
+    url = url.join('.')
+    name + '@' + url
+  end
+
+  def self.generate_url(options = {})
+    ext = random_from(URL_SUFFIXES)
+    ext += '.' + random_from(URL_COUNTRY_CODES) if rand(2) > 0
+    "http://www.#{generate_alnum_word}.#{ext}/"
+  end
+
+  def self.generate_markdown(length = 100..1000, options = {})
+    @footnote_num = 0
+    if length.is_a?(Range)
+      raise ArgumentError, "range must be positive" unless length.min
+      length = length.min + rand(length.max - length.min)
+    end
+    text = []
+    while length > 0
+      case rand
+      when 0.3..1 # generate paragraph
+        l = gen_num(length, 50)
+        paragraph = gen_md_para(l)
+        if rand > 0.95 # make block quote
+          paragraph = '> ' + paragraph
+        end
+        text << paragraph
+        length -= l
+      when 0.2..0.3 # generate list
+        li = random_from(MARKDOWN_LIST_MARKERS) + ' '
+        num_elements = gen_num(length/5, 10)
+        num_elements.times do
+          break if length == 0
+          if rand > 0.75 # do paragraph list element
+            xli = li
+            (2 + rand(3)).times do |i|
+              break if length == 0
+              l = gen_num(length, 10)
+              text << xli
+              text << gen_md_para(l, :no_footnotes => true)
+              text << "\n\n"
+              xli = ' ' * xli.size if i == 0
+              length -= l
+            end
+          else
+            l = gen_num(length, 10)
+            text << li
+            text << gen_md_para(l, :no_footnotes => true)
+            text << "\n"
+            length -= l
+          end
+        end
+      when 0.1..0.2 # header
+        l = gen_num(length, 7)
+        t = gen_md_para(l, :no_footnotes => true)
+        if rand > 0.8
+          t += "\n" + random_from(%w{= -}) * t.size
+        else
+          t = ('#' * (1 + rand(6))) + ' ' + t
+        end
+        length -= l
+        text << t
+      else 
+        text << '---'
+      end
+      text << "\n\n"
+    end
+    text.join()
+  end
+
+  def self.random_word
+    random_from(WORDS)
+  end
+
+  def self.random_char
+    random_from(CHARS)
+  end
+
+  def self.random_alnum
+    random_from(ALNUM)
+  end
+
+  def self.random_alpha
+    random_from(ALPHA)
+  end
+
+  private
+  
+  def self.gen_md_para(length, options = {})
+    link_words = rand(1 + length/10)
+    length -= link_words
+    text = gen_md_text(length)
+    text << "\n"
+    footnote_cnt = 0
+    while link_words > 0
+      if options[:no_footnotes] or rand > 0.5
+        if rand > 0.6 # inline link
+          l = gen_num(link_words, 5)
+          link = "[#{gen_md_text(l)}](#{generate_url} \"#{generate_text(1 + rand(5))}\")"
+          text.insert(rand(text.length - footnote_cnt), link)
+          link_words -= l
+        else          # auto link
+          text.insert(rand(text.length - footnote_cnt), "<#{generate_url}>")
+          link_words -= 1
+        end
+      else            # footnote link
+        l = gen_num(link_words, 5)
+        reference = "[#{gen_md_text(l).join(' ')}][#{@footnote_num}]"
+        text.insert(rand(text.length - footnote_cnt), reference)
+        text << link = "\n[#{@footnote_num}]: #{generate_url} \"#{generate_text(1 + rand(5))}\""
+        @footnote_num += 1
+        footnote_cnt += 1
+        link_words -= l
+      end
+    end
+    text.pop if text.last == "\n"
+    text.join(' ')
+  end
+
+  def self.gen_md_text(length)
+    text = Array.new(length) {|x| random_word}
+    if rand > 0.8
+      (1 + rand(Math.sqrt(length))).times do
+        first = rand(text.size)
+        last = first + rand(3)
+        last = text.size - 1 if last >= text.size
+        words = text.slice!(first..last)
+        em = random_from(MARKDOWN_EMPHASIS_MARKERS)
+        words = "#{em}#{words.join(' ')}#{em}" unless words.join.index(em[0,1])
+        text.insert(first, words).flatten!
+      end
+    end
+    text
+  end
+
+  def self.gen_num(max1, max2)
+    minmax = [max1, max2].min
+    return minmax == 0 ? 0 : 1 + rand(minmax)
+  end
+
+  def self.random_from(list)
+    list[rand(list.size)]
+  end
+end